You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2026/05/27 13:58:15 UTC
(tika) branch main updated: TIKA-4731 - improve charset detection and junk detection (#2839)
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new a2bc3513ac TIKA-4731 - improve charset detection and junk detection (#2839)
a2bc3513ac is described below
commit a2bc3513ac9c2dd6d0f63ea37363c3a3ec172cb4
Author: Tim Allison <ta...@apache.org>
AuthorDate: Wed May 27 09:58:09 2026 -0400
TIKA-4731 - improve charset detection and junk detection (#2839)
---
.skills/tika-eval-h2-query.md | 92 ++
.../integration-testing/tika-eval-regression.adoc | 364 +++++
.../apache/tika/ml/chardetect/AdaptiveProbe.java | 80 +
.../tika/ml/chardetect/CharsetConfusables.java | 27 +
.../tika/ml/chardetect/HtmlByteStripper.java | 198 ++-
.../ml/chardetect/MojibusterEncodingDetector.java | 198 ++-
.../NaiveBayesBigramEncodingDetector.java | 488 +++++-
.../org/apache/tika/ml/chardetect/nb-bigram.bin | Bin 975490 -> 1016638 bytes
.../tika/ml/chardetect/AdaptiveProbeTest.java | 118 ++
.../apache/tika/ml/chardetect/CalibrateTopK.java | 353 ++++
.../apache/tika/ml/chardetect/CheckUtf8OnFile.java | 83 +
.../tika/ml/chardetect/HtmlByteStripperTest.java | 245 +++
.../ml/chardetect/InspectBigramContributions.java | 221 +++
.../apache/tika/ml/chardetect/TraceMojibuster.java | 233 +++
.../chardetect/tools/BuildCharsetTrainingData.java | 23 +-
.../chardetect/tools/DiagnoseDiscrimination.java | 399 +++++
.../chardetect/tools/RebalanceCharsetTraining.java | 209 +++
.../ml/chardetect/tools/TrainNaiveBayesBigram.java | 56 +-
tika-ml/tika-ml-junkdetect/pom.xml | 12 +
.../{V7Tables.java => BigramTables.java} | 36 +-
.../tika/ml/junkdetect/HtmlContentCleaner.java | 108 ++
.../apache/tika/ml/junkdetect/JunkDetector.java | 842 ++++++----
.../ml/junkdetect/JunkFilterEncodingDetector.java | 274 ++--
.../tika/ml/junkdetect/TextQualityFeatures.java | 608 +++++++
.../ml/junkdetect/tools/AnalyzeHanByBlock.java | 201 ---
.../ml/junkdetect/tools/BuildJunkTrainingData.java | 7 +
.../ml/junkdetect/tools/CountPerScriptBigrams.java | 326 ----
.../tika/ml/junkdetect/tools/EvalJunkDetector.java | 777 ---------
.../junkdetect/tools/EvalJunkOnCharsetDevtest.java | 688 --------
.../tools/JunkDetectorTrainingConfig.java | 9 +-
.../junkdetect/tools/PrototypeCodepointHash.java | 1208 --------------
.../tika/ml/junkdetect/tools/TrainJunkModel.java | 1701 ++++++++++----------
.../org/apache/tika/ml/junkdetect/junkdetect.bin | Bin 2810396 -> 2321862 bytes
.../apache/tika/ml/junkdetect/EntityRefProbe.java | 164 --
...rV7Test.java => JunkDetectorRoundTripTest.java} | 165 +-
.../tika/ml/junkdetect/JunkDetectorSmokeTest.java | 7 +-
.../junkdetect/JunkFilterEncodingDetectorTest.java | 43 +
.../ml/junkdetect/LatinSiblingComparisonTest.java | 141 ++
.../ml/junkdetect/TextQualityFeaturesTest.java | 201 +++
.../apache/tika/ml/junkdetect/TraceJunkFilter.java | 536 ++++++
.../tools/BuildJunkAugmentationData.java | 862 ++++++++++
.../tools/BuildJunkAugmentationDataTest.java | 429 +++++
.../tools/JunkDetectorTrainingConfigTest.java | 5 +-
.../apache/tika/parser/pkg/PackageParserTest.java | 4 +
.../java/org/apache/tika/parser/pkg/ZipParser.java | 29 +-
45 files changed, 7914 insertions(+), 4856 deletions(-)
diff --git a/.skills/tika-eval-h2-query.md b/.skills/tika-eval-h2-query.md
new file mode 100644
index 0000000000..d4f0b2c378
--- /dev/null
+++ b/.skills/tika-eval-h2-query.md
@@ -0,0 +1,92 @@
+# Query the tika-eval H2 database directly
+
+`tika-eval` (Compare / Profile / Report) stores everything in an **H2** database
+— the `-d <name>` you pass to `Compare` produces `<name>.mv.db`. The xlsx/`summary.md`
+reports only surface pre-canned views; for anything else (exact counts, custom
+joins, "better vs worse" tallies the reports don't compute) connect to the H2 db
+and run SQL.
+
+## Connecting (the part that trips people up)
+
+tika-eval creates the db with **no username and no password** (`H2Util` calls
+`DriverManager.getConnection(url)` with no creds). So:
+
+- **URL:** `jdbc:h2:<absolute-path-without-the-.mv.db-suffix>`
+- **user / password: empty.** `-user sa` (H2's old default) **fails** with
+ "Wrong user name or password" — the db wasn't created with `sa`.
+- Add `;IFEXISTS=TRUE` so a typo opens nothing instead of silently creating a new
+ empty db, and `;ACCESS_MODE_DATA=r` for a safe read-only open.
+- **H2 is single-writer**: don't query while a `Compare`/`Profile`/`Report` run
+ has the db open (file lock). Query after the run finishes.
+
+Use the `h2-*.jar` that ships with tika-eval-app (in its `target/dependency/` or
+the unzipped runtime `lib/`).
+
+### One-shot query (H2 Shell)
+
+```bash
+H2=path/to/h2-*.jar
+DB='jdbc:h2:path/to/eval;IFEXISTS=TRUE;ACCESS_MODE_DATA=r' # note: no .mv.db
+java -cp "$H2" org.h2.tools.Shell -url "$DB" -user '' -password '' \
+ -sql "SHOW TABLES"
+```
+
+Pass the SQL as a **single line** to `-sql` (a multi-line arg confuses the Shell;
+it then waits on stdin and appears to hang).
+
+## Key tables
+
+| table | what's in it |
+|---|---|
+| `PROFILES_A` / `PROFILES_B` | one row per extracted file: `FILE_NAME`, `MD5`, `MIME_ID`, `CONTAINER_ID`, `EMBEDDED_FILE_PATH`, `LENGTH`, `NUM_PAGES`, … (A = "before"/-a, B = "after"/-b) |
+| `CONTENTS_A` / `CONTENTS_B` | text profile per file (join on `ID`): `OOV`, `LANGUAGENESS`, `NUM_TOKENS`, `NUM_COMMON_TOKENS`, `LANG_ID_1`/`LANG_ID_PROB_1`, `TOKEN_ENTROPY_RATE`, … |
+| `CONTENT_COMPARISONS` | per-file A↔B comparison (`ID`): `DICE_COEFFICIENT`, `OVERLAP`, top token diffs |
+| `MIMES` | `MIME_ID` → `MIME_STRING` |
+| `CONTAINERS` | container id → input file path |
+| `*_COMPARED` (`EXCEPTIONS_COMPARED`, `PARSE_TIME_COMPARED`) | pre-joined A/B summaries |
+
+## Example queries
+
+Detected charset distribution in the "after" run (fast):
+
+```sql
+SELECT m.MIME_STRING, COUNT(*) n
+FROM PROFILES_B p JOIN MIMES m ON p.MIME_ID = m.MIME_ID
+GROUP BY m.MIME_STRING ORDER BY n DESC LIMIT 20;
+```
+
+Worst content divergences (already A/B-paired, fast):
+
+```sql
+SELECT ID, DICE_COEFFICIENT FROM CONTENT_COMPARISONS
+WHERE DICE_COEFFICIENT < 0.5 ORDER BY DICE_COEFFICIENT;
+```
+
+OOV / languageness better-vs-worse tally. **A and B are paired by `id`** — the
+same row `id` is the same file in both runs (this is exactly how tika-eval's own
+reports join: `join profiles_b pb on pa.id = pb.id`). Join on `id`, never on
+container/path — that's a fast PK join:
+
+```sql
+SELECT SUM(CASE WHEN cb.OOV < ca.OOV THEN 1 ELSE 0 END) AS oov_better,
+ SUM(CASE WHEN cb.OOV > ca.OOV THEN 1 ELSE 0 END) AS oov_worse
+FROM CONTENTS_A ca JOIN CONTENTS_B cb ON ca.ID = cb.ID;
+```
+
+Net common-tokens A vs B — the headline "more real text recovered" metric (a big
+positive delta is the usual sign of better charset/extraction in B):
+
+```sql
+SELECT SUM(ca.NUM_COMMON_TOKENS) AS common_a,
+ SUM(cb.NUM_COMMON_TOKENS) AS common_b,
+ SUM(cb.NUM_COMMON_TOKENS) - SUM(ca.NUM_COMMON_TOKENS) AS delta
+FROM CONTENTS_A ca JOIN CONTENTS_B cb ON ca.ID = cb.ID;
+```
+
+To bring in mime/path, join `PROFILES_A pa ON pa.ID = ca.ID` (and `pb`/`cc`
+likewise on the same `id`) — all on `id`.
+
+## Tip
+
+For a quick interactive session, drop `-sql` and you get an H2 prompt; `SHOW
+COLUMNS FROM <table>;` lists a table's columns.
diff --git a/docs/modules/ROOT/pages/advanced/integration-testing/tika-eval-regression.adoc b/docs/modules/ROOT/pages/advanced/integration-testing/tika-eval-regression.adoc
new file mode 100644
index 0000000000..24460db449
--- /dev/null
+++ b/docs/modules/ROOT/pages/advanced/integration-testing/tika-eval-regression.adoc
@@ -0,0 +1,364 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+= Regression evaluation with tika-app + tika-eval
+
+A step-by-step recipe for comparing two Tika configurations against the
+same corpus. Typical uses:
+
+* Verify a code change or dependency bump didn't regress extraction.
+* Compare two parser/detector configurations (e.g. different encoding-detector
+ chains) head-to-head.
+* Generate signed-off reports for a release.
+
+== Concept
+
+The flow is three stages:
+
+. *Extract A* — run Tika config `A` over the corpus, writing one JSON
+ per input file to `extracts-A/`.
+. *Extract B* — run Tika config `B` over the same corpus, writing to
+ `extracts-B/`.
+. *Compare + Report* — run `tika-eval` to load both extract trees into
+ a database and produce `.xlsx` reports highlighting differences
+ (MIME, content, exceptions, embedded documents, metadata).
+
+Both extracts include SHA-256 digests so `tika-eval` can align
+embedded documents across runs even when filenames differ.
+
+== Prerequisites
+
+[source,bash]
+----
+# Build tika-app and tika-eval (in the tika source tree)
+./mvnw install -pl tika-app,tika-eval/tika-eval-app -am -DskipTests \
+ -Dmaven.repo.local=$(pwd)/.local_m2_repo
+----
+
+This produces two distribution zips:
+
+* `tika-app/target/tika-app-{tika-version}.zip` — contains the
+ tika-app jar, `lib/` (dependencies), and `plugins/` (file-system
+ fetcher/emitter/iterator pipes plugin). Unzip somewhere and run
+ from that directory; `TikaAsyncCLI` auto-discovers the `plugins/`
+ sibling, so no extra flag is needed.
+* `tika-eval/tika-eval-app/target/tika-eval-app-{tika-version}.zip` —
+ contains the tika-eval-app jar and its `lib/`. Unzip and run from
+ there.
+
+[source,bash]
+----
+mkdir -p ~/tika-runtime && cd ~/tika-runtime
+unzip -q tika-app/target/tika-app-{tika-version}.zip -d tika-app
+unzip -q tika-eval/tika-eval-app/target/tika-eval-app-{tika-version}.zip -d tika-eval
+----
+
+Running the bare `tika-app-*.jar` or `tika-eval-app-*.jar` (without
+the unzipped `lib/` siblings) will fail at runtime with
+`ClassNotFoundException` — the jars are intentionally not fat jars.
+
+== Step 1: prepare the corpus
+
+Put the input files somewhere on disk. For this example we'll use
+`/data/corpus/`. Tika's pipes iterator walks the tree recursively;
+files at any depth are processed.
+
+== Step 2: write the two extraction configs
+
+When you pass `-i <input>` and `-o <output>` on the command line,
+tika-app auto-builds the fetcher / emitter / pipes-iterator for the
+file system. The config only needs the things you want to OVERRIDE
+from defaults — typically just the digester and (for this kind of
+A/B test) the field under test.
+
+The two configs should differ in exactly *one* thing — the field
+under test. Everything else (digester, parsers, worker count) should
+match.
+
+=== Config A (baseline)
+
+`tika-config-A.json`:
+
+[source,json]
+----
+{
+ "parse-context": {
+ "commons-digester-factory": {
+ "digests": [{ "algorithm": "SHA256" }],
+ "skipContainerDocumentDigest": false
+ }
+ }
+}
+----
+
+The `parse-context > commons-digester-factory` block makes every
+extracted record carry an `X-TIKA:digest:SHA256` metadata field —
+required by `tika-eval` for embedded-document alignment across runs.
+
+=== Config B (variant under test)
+
+Copy config A, add the field you're testing. Example — test a
+different encoding-detector chain:
+
+[source,json]
+----
+{
+ "encoding-detectors": [
+ { "bom-detector": {} },
+ { "html-encoding-detector": {} },
+ { "mojibuster-encoding-detector": {} },
+ { "junk-filter-encoding-detector": {} }
+ ],
+ "parse-context": {
+ "commons-digester-factory": {
+ "digests": [{ "algorithm": "SHA256" }],
+ "skipContainerDocumentDigest": false
+ }
+ }
+}
+----
+
+When an explicit `encoding-detectors` array is present, TikaLoader
+uses it verbatim instead of the SPI-discovered default chain. Other
+encoding detectors on the classpath are excluded.
+
+If you need to override anything else (parsers, plugin-roots, content
+handler factory), put it under additional top-level keys. Anything
+not specified takes its default.
+
+== Step 3: run extractions
+
+[source,bash]
+----
+cd ~/tika-runtime/tika-app
+
+# Extract A — baseline
+java -jar tika-app-{tika-version}.jar \
+ -i /data/corpus \
+ -o /data/extracts-A \
+ -c /path/to/tika-config-A.json \
+ -n 8
+
+# Extract B — variant
+java -jar tika-app-{tika-version}.jar \
+ -i /data/corpus \
+ -o /data/extracts-B \
+ -c /path/to/tika-config-B.json \
+ -n 8
+----
+
+Flags:
+
+* `-i` / `--inputDir` — input directory (recursive walk)
+* `-o` / `--outputDir` — output directory for per-file JSON
+* `-c` / `--config` — optional config overrides
+* `-n` / `--numClients` — forked worker processes (default 4)
+* `-p` / `--pluginsDir` — explicit plugins directory (omit when
+ running from the unzipped distribution — auto-discovery handles it)
+* `-T` / `--timeoutMs` — per-parse timeout
+
+Each run forks worker processes and produces one `.json` per input
+file under the output directory. Run them sequentially or in
+parallel — they're independent.
+
+Progress is logged to stderr. On a typical workstation, extraction
+throughput is in the hundreds-of-files-per-minute range, depending on
+parser cost and worker count.
+
+== Step 4: compare and report
+
+[source,bash]
+----
+java -jar tika-eval/tika-eval-app/target/tika-eval-app-{tika-version}.jar \
+ -n 8 -r -d my-comparison \
+ -a /data/extracts-A \
+ -b /data/extracts-B
+----
+
+The `Compare` subcommand keyword is optional — the CLI infers it from
+the `-a` / `-b` flags. The `-r` flag both runs the Report stage and
+zips the resulting reports directory for easy archiving.
+
+Options:
+
+* `-a` / `--extractsA` — A's extract directory (required)
+* `-b` / `--extractsB` — B's extract directory (required)
+* `-i` / `--inputDir` — original binary input directory (optional, lets
+ tika-eval pair extracts to source files even if A or B failed on some)
+* `-d` / `--db` — H2 database name/path. A short label is fine —
+ tika-eval will create `{label}.mv.db` and a `{label}-reports/` dir
+ alongside. Persist the db if you want to re-run Report later.
+* `-r` / `--report` — automatically run the Report step after Compare,
+ and zip the reports directory.
+* `-rd` / `--reportsDir` — explicit reports directory (overrides the
+ default derived from `-d`).
+* `-n` / `--numWorkers` — comparison worker count.
+* `-c` / `--config` — optional tika-eval JSON config.
+
+== Step 5: read the reports
+
+The reports directory contains subdirectories:
+
+* `mimes/` — MIME type comparison. Headline files:
+** `mime_diffs_A_to_B.xlsx` — summary counts per (A type, B type)
+ transition
+** `mime_diffs_A_to_B_details.xlsx` — one row per file that changed
+* `content/` — token-count and content comparison
+* `exceptions/` — new/changed exceptions in B vs A
+* `attachments/` — embedded-document comparison
+* `metadata/` — metadata comparison
+
+Open the `.xlsx` files directly, or use the `regression` skill
+(`~/.claude/skills/regression/SKILL.md`) for guided triage.
+
+== Tips
+
+* *Keep the digester identical between A and B.* tika-eval uses the
+ `X-TIKA:digest:SHA256` field on embedded documents to align records
+ across the two extracts. If A digests and B doesn't (or different
+ algorithms), the embedded-doc alignment falls back to filename and
+ produces false-positive diffs.
+
+* *Use the same `numClients` and `timeoutMillis`*. Different parallelism
+ can produce different timeout behavior that shows up as spurious
+ exception diffs.
+
+* *Run on copies of the binaries when investigating crashes.* Some
+ parsers may corrupt input files in rare cases; keep a known-good
+ source tree separate from working copies.
+
+* *Persist the H2 database (`-d /path/to/eval.db`)* if you want to
+ re-run the Report step or query it directly with the H2 console.
+ Omitting `-d` writes to a temp file that's cleaned up on exit.
+
+* *Re-run only Report*: if you already have the database, skip Compare
+ and run the Report tool directly:
++
+[source,bash]
+----
+java -jar tika-eval/tika-eval-app/target/tika-eval-app-{tika-version}.jar Report \
+ -d /tmp/eval.db -rd /data/reports/
+----
+
+== Worked example: encoding-detector chain comparison
+
+Compare the SPI default detector chain (A) against an explicit
+`[BOM, HTML, Mojibuster, JunkFilter]` chain (B) over a 29 000-file
+Common Crawl HTML sample at `~/data/commoncrawl/cc-html-sample`.
+
+. One-time build: `./mvnw install -pl tika-app,tika-eval/tika-eval-app -am -DskipTests`
+. Unzip both distributions: `unzip tika-app/target/tika-app-*.zip` and
+ `unzip tika-eval/tika-eval-app/target/tika-eval-app-*.zip`.
+. Write `tika-config-A.json` (digester only) and `tika-config-B.json`
+ (digester + the four-detector `encoding-detectors` array) per the
+ examples above.
+. Extract A:
++
+[source,bash]
+----
+cd ~/tika-runtime/tika-app
+java -jar tika-app-{tika-version}.jar \
+ -i ~/data/commoncrawl/cc-html-sample \
+ -o ~/data/extracts/cc-html-sample-A \
+ -c ~/configs/tika-config-A.json -n 8
+----
+. Extract B (same form, different config + output):
++
+[source,bash]
+----
+java -jar tika-app-{tika-version}.jar \
+ -i ~/data/commoncrawl/cc-html-sample \
+ -o ~/data/extracts/cc-html-sample-B \
+ -c ~/configs/tika-config-B.json -n 8
+----
+. Compare + report:
++
+[source,bash]
+----
+cd ~/tika-runtime/tika-eval
+java -jar tika-eval-app-{tika-version}.jar \
+ -n 8 -r -d cc-html-29k-A-vs-B \
+ -a ~/data/extracts/cc-html-sample-A \
+ -b ~/data/extracts/cc-html-sample-B
+----
++
+Produces `cc-html-29k-A-vs-B-reports/` plus a `.tar.gz` of the same
+alongside `cc-html-29k-A-vs-B.mv.db`.
+
+For a 29 K-file HTML sample on a typical workstation (8 forked workers,
+i9-ish CPU): each extraction completes in roughly 45-60 seconds; the
+Compare step finishes in 2-5 minutes depending on extract size.
+. Open `reports/mimes/mime_diffs_A_to_B.xlsx` to see the headline
+ MIME-detection differences; the encoding-detector chain change
+ surfaces as charset diffs in `mimes/mime_diffs_A_to_B_details.xlsx`
+ (`X-TIKA:detected_encoding`).
+
+A regression analysis writeup goes in
+`~/Desktop/claude-todo/<reports-dir-name>-analysis.md` per the
+`regression` skill.
+
+== Querying the H2 database directly
+
+`tika-eval` stores everything in an *H2* database — the `-d <name>` passed to
+`Compare` produces `<name>.mv.db`. The generated reports surface only pre-canned
+views; for custom counts and joins, connect to the H2 file and run SQL.
+
+The database is created with *no username and no password*
+(`H2Util` opens it with `DriverManager.getConnection(url)` and no credentials).
+So the old H2 default `-user sa` *fails* with "Wrong user name or password" — use
+empty credentials. Append `;IFEXISTS=TRUE` (so a typo opens nothing rather than
+creating a new empty db) and `;ACCESS_MODE_DATA=r` for a read-only open. H2 is
+single-writer: query only after the `Compare`/`Profile`/`Report` run has released
+the file lock. Use the `h2-*.jar` shipped with `tika-eval-app`.
+
+[source,bash]
+----
+H2=path/to/h2-x.y.z.jar
+# note: omit the .mv.db suffix in the URL
+DB='jdbc:h2:path/to/eval-db;IFEXISTS=TRUE;ACCESS_MODE_DATA=r'
+java -cp "$H2" org.h2.tools.Shell -url "$DB" -user '' -password '' \
+ -sql "SHOW TABLES"
+----
+
+Pass SQL to `-sql` as a *single line* (a multi-line argument leaves the Shell
+waiting on stdin).
+
+Key tables: `profiles_a`/`profiles_b` (one row per extracted file: `file_name`,
+`mime_id`, `length`, …), `contents_a`/`contents_b` (text profile: `oov`,
+`languageness`, `num_tokens`, `lang_id_1`, …), `content_comparisons`
+(`dice_coefficient`, `overlap`), `mimes`, `containers`. *A and B are paired by
+`id`* — the same row `id` is the same file in both runs (this is how the built-in
+reports join: `join profiles_b pb on pa.id = pb.id`). Always join on `id`.
+
+[source,sql]
+----
+-- OOV / languageness: how many files improved vs regressed in B (fast PK join)
+SELECT SUM(CASE WHEN cb.oov < ca.oov THEN 1 ELSE 0 END) AS oov_better,
+ SUM(CASE WHEN cb.oov > ca.oov THEN 1 ELSE 0 END) AS oov_worse
+FROM contents_a ca JOIN contents_b cb ON ca.id = cb.id;
+
+-- net common-tokens A vs B (headline "more real text recovered" metric)
+SELECT SUM(ca.num_common_tokens) AS common_a,
+ SUM(cb.num_common_tokens) AS common_b,
+ SUM(cb.num_common_tokens) - SUM(ca.num_common_tokens) AS delta
+FROM contents_a ca JOIN contents_b cb ON ca.id = cb.id;
+
+-- detected charset distribution in the B run
+SELECT m.mime_string, COUNT(*) n
+FROM profiles_b p JOIN mimes m ON p.mime_id = m.mime_id
+GROUP BY m.mime_string ORDER BY n DESC;
+----
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/AdaptiveProbe.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/AdaptiveProbe.java
new file mode 100644
index 0000000000..34a081ec32
--- /dev/null
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/AdaptiveProbe.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.chardetect;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.commons.io.IOUtils;
+
+import org.apache.tika.io.TikaInputStream;
+
+/**
+ * Reads an encoding-detection probe sized by <em>content</em>, not raw bytes.
+ *
+ * <p>A fixed raw probe (e.g. 16 KB) starves the detectors when a page
+ * leads with a large {@code <head>}/inline script: after tag-stripping there's
+ * little body text, and the bytes that distinguish charsets sit past the
+ * window. This grows the read until ~{@code contentTarget} bytes of
+ * tag-stripped content are present, capped at {@code rawCap} raw bytes.
+ *
+ * <p>Text-rich pages stop early (~one chunk); markup-heavy pages read deeper,
+ * bounded by the cap. Multi-byte encodings (UTF-16/32) register no ASCII tags
+ * so they stop at {@code contentTarget} raw bytes.
+ */
+public final class AdaptiveProbe {
+
+ /** Default body-content target. */
+ public static final int DEFAULT_CONTENT_TARGET = 16384;
+ /** Default hard ceiling on raw bytes read. */
+ public static final int DEFAULT_RAW_CAP = 524288;
+
+ private AdaptiveProbe() {
+ }
+
+ /**
+ * Reads from {@code tis} (mark/reset preserved) until tag-stripped content
+ * reaches {@code contentTarget}, the raw read reaches {@code rawCap}, or
+ * EOF — whichever first. Returns the raw bytes read.
+ */
+ public static byte[] read(TikaInputStream tis, int contentTarget, int rawCap)
+ throws IOException {
+ tis.mark(rawCap);
+ try {
+ byte[] buf = new byte[rawCap];
+ byte[] stripDst = new byte[rawCap];
+ int total = 0;
+ while (total < rawCap) {
+ int want = Math.min(rawCap - total, contentTarget);
+ int n = IOUtils.read(tis, buf, total, want);
+ total += n;
+ HtmlByteStripper.Result r =
+ HtmlByteStripper.stripTags(buf, 0, total, stripDst, 0);
+ int content = r.tagCount > 0 ? r.length : total;
+ if (content >= contentTarget || n < want) {
+ break; // enough body text, or EOF
+ }
+ }
+ if (total == 0) {
+ return new byte[0];
+ }
+ return total == rawCap ? buf : Arrays.copyOf(buf, total);
+ } finally {
+ tis.reset();
+ }
+ }
+}
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/CharsetConfusables.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/CharsetConfusables.java
index 12d9914292..c3caace45d 100644
--- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/CharsetConfusables.java
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/CharsetConfusables.java
@@ -154,6 +154,31 @@ public final class CharsetConfusables {
*/
public static final Set<String> SBCS_LATIN_FAMILY;
+ /**
+ * Strict subset of {@link #SBCS_LATIN_FAMILY} containing only the
+ * Western European Latin members. These are the SBCS classes that
+ * share win-1252's source corpus (English / German / French /
+ * Spanish / Italian / Dutch / Portuguese / etc.); when NB picks
+ * one of these as top-1 on a probe, the model has a measurable
+ * bias the chain can fail to rescue, and rewriting to win-1252
+ * is justified because the underlying text is presumed to be
+ * Western European.
+ *
+ * <p><b>Deliberately excludes</b>: windows-1250 / ISO-8859-2 /
+ * IBM852 (Central European), windows-1257 / ISO-8859-13 (Baltic),
+ * windows-1254 (Turkish), ISO-8859-3 (Maltese), ISO-8859-4
+ * (Northern European), ISO-8859-9 (legacy Turkish), ISO-8859-16
+ * (Romanian / Albanian). These represent different language
+ * regions; "rewrite to win-1252" would corrupt their content.</p>
+ *
+ * <p>Note: this set is intentionally limited to the trained-by-NB
+ * classes that overlap win-1252's source text. ISO-8859-1 is
+ * also Western European but is not a trained NB class (win-1252
+ * is its superset), so NB never emits it as top-1 — no need to
+ * list it here.</p>
+ */
+ public static final Set<String> WESTERN_LATIN_FAMILY;
+
static {
// ----------------------------------------------------------------
// Symmetric groups
@@ -298,6 +323,8 @@ public final class CharsetConfusables {
"ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4",
"ISO-8859-9", "ISO-8859-13", "ISO-8859-15", "ISO-8859-16",
"x-MacRoman")));
+ WESTERN_LATIN_FAMILY = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
+ "x-MacRoman", "IBM850", "ISO-8859-15")));
}
private CharsetConfusables() {
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/HtmlByteStripper.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/HtmlByteStripper.java
index 1443ae9723..37ae5ad26b 100644
--- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/HtmlByteStripper.java
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/HtmlByteStripper.java
@@ -66,6 +66,29 @@ public final class HtmlByteStripper {
* end at the next {@code >}. Internal subsets ({@code <!DOCTYPE foo [ ... ]>})
* are rare; we'd stop at the first nested {@code >}. Acceptable. */
private static final int DECL_OR_PI = 10;
+ /** Just saw {@code &}. Next byte decides whether this is a named
+ * entity ({@code &}), a numeric reference ({@code ©} or
+ * {@code ©}), or a stray ampersand. */
+ private static final int ENTITY = 11;
+ private static final int ENTITY_NAME = 12;
+ /** Just saw {@code &#}. Next byte: {@code x}/{@code X} for hex, or
+ * decimal digit for decimal. */
+ private static final int ENTITY_NUM = 13;
+ private static final int ENTITY_DEC = 14;
+ private static final int ENTITY_HEX = 15;
+
+ /**
+ * Maximum body length for an entity (bytes after {@code &}, including
+ * any {@code #}/{@code x} prefix, excluding the trailing {@code ;}).
+ * Standard HTML5 named entities are at most 32 bytes
+ * ({@code ∳}), but only a few dozen
+ * exceed 16; 99% of real-world entities are well under that. A 16-byte
+ * cap covers the common cases ({@code }, {@code «},
+ * {@code Á}, {@code …}, {@code ·}, etc.) and
+ * bounds pathological input that might otherwise eat language-content
+ * bytes before bailing out.
+ */
+ private static final int MAX_ENTITY_BODY_LEN = 16;
private static final byte[] SCRIPT = {'s', 'c', 'r', 'i', 'p', 't'};
private static final byte[] STYLE = {'s', 't', 'y', 'l', 'e'};
@@ -105,10 +128,13 @@ public final class HtmlByteStripper {
public final int length;
/** Number of well-formed tags parsed (including comments). */
public final int tagCount;
+ /** Number of well-formed HTML entities stripped from TEXT. */
+ public final int entityCount;
- public Result(int length, int tagCount) {
+ public Result(int length, int tagCount, int entityCount) {
this.length = length;
this.tagCount = tagCount;
+ this.entityCount = entityCount;
}
}
@@ -133,6 +159,33 @@ public final class HtmlByteStripper {
*/
public static Result strip(byte[] src, int srcOffset, int srcLen,
byte[] dst, int dstOffset) {
+ // Back-compat alias for charset-detection callers (Mojibuster,
+ // training tools): strips tags AND well-formed entities.
+ return stripTagsAndEntities(src, srcOffset, srcLen, dst, dstOffset);
+ }
+
+ /**
+ * Strip tags <em>and</em> well-formed HTML entities. For <b>charset
+ * detection</b> (Mojibuster): entities are ASCII, charset-neutral, and
+ * bias the byte-bigram statistics, so they're removed.
+ */
+ public static Result stripTagsAndEntities(byte[] src, int srcOffset, int srcLen,
+ byte[] dst, int dstOffset) {
+ return strip(src, srcOffset, srcLen, dst, dstOffset, true);
+ }
+
+ /**
+ * Strip tags only; entities pass through unchanged. For <b>junk
+ * detection</b>, which expands them in string space afterward (dropping
+ * them here would make that expansion a no-op).
+ */
+ public static Result stripTags(byte[] src, int srcOffset, int srcLen,
+ byte[] dst, int dstOffset) {
+ return strip(src, srcOffset, srcLen, dst, dstOffset, false);
+ }
+
+ private static Result strip(byte[] src, int srcOffset, int srcLen,
+ byte[] dst, int dstOffset, boolean dropEntities) {
int w = dstOffset;
int state = TEXT;
int nameStart = 0;
@@ -140,6 +193,11 @@ public final class HtmlByteStripper {
int rawMatch = 0;
int end = srcOffset + srcLen;
int tagCount = 0;
+ int entityCount = 0;
+ // Position of the leading '&' for the in-progress entity.
+ // Tracked so the bailout path can emit the consumed prefix
+ // as literal text when the parse fails (e.g. "AT&T").
+ int entityStart = 0;
int attrNameStart = 0;
// When true, the current quoted attribute value's bytes are
// emitted to dst (attribute name matched TEXT_ATTRS). Reset
@@ -152,6 +210,9 @@ public final class HtmlByteStripper {
case TEXT:
if (b == '<') {
state = LT;
+ } else if (b == '&' && dropEntities) {
+ state = ENTITY;
+ entityStart = i;
} else {
dst[w++] = b;
}
@@ -189,6 +250,121 @@ public final class HtmlByteStripper {
}
break;
+ case ENTITY:
+ // First byte after '&' decides path.
+ if (b == '#') {
+ state = ENTITY_NUM;
+ } else if (isAsciiLetter(b)) {
+ state = ENTITY_NAME;
+ } else {
+ // Not entity-shaped: emit consumed prefix (just '&')
+ // and re-process b under TEXT semantics.
+ for (int k = entityStart; k < i; k++) {
+ dst[w++] = src[k];
+ }
+ if (b == '<') {
+ state = LT;
+ } else if (b == '&') {
+ entityStart = i;
+ // state stays ENTITY
+ } else {
+ dst[w++] = b;
+ state = TEXT;
+ }
+ }
+ break;
+
+ case ENTITY_NAME:
+ if (b == ';') {
+ entityCount++;
+ state = TEXT;
+ } else if (isAsciiLetter(b)
+ && (i - entityStart) <= MAX_ENTITY_BODY_LEN) {
+ // continue accumulating; no emit
+ } else {
+ for (int k = entityStart; k < i; k++) {
+ dst[w++] = src[k];
+ }
+ if (b == '<') {
+ state = LT;
+ } else if (b == '&') {
+ entityStart = i;
+ state = ENTITY;
+ } else {
+ dst[w++] = b;
+ state = TEXT;
+ }
+ }
+ break;
+
+ case ENTITY_NUM:
+ // First byte after '&#': 'x'/'X' for hex, digit for decimal.
+ if (b == 'x' || b == 'X') {
+ state = ENTITY_HEX;
+ } else if (isAsciiDigit(b)) {
+ state = ENTITY_DEC;
+ } else {
+ for (int k = entityStart; k < i; k++) {
+ dst[w++] = src[k];
+ }
+ if (b == '<') {
+ state = LT;
+ } else if (b == '&') {
+ entityStart = i;
+ state = ENTITY;
+ } else {
+ dst[w++] = b;
+ state = TEXT;
+ }
+ }
+ break;
+
+ case ENTITY_DEC:
+ if (b == ';') {
+ entityCount++;
+ state = TEXT;
+ } else if (isAsciiDigit(b)
+ && (i - entityStart) <= MAX_ENTITY_BODY_LEN) {
+ // continue
+ } else {
+ for (int k = entityStart; k < i; k++) {
+ dst[w++] = src[k];
+ }
+ if (b == '<') {
+ state = LT;
+ } else if (b == '&') {
+ entityStart = i;
+ state = ENTITY;
+ } else {
+ dst[w++] = b;
+ state = TEXT;
+ }
+ }
+ break;
+
+ case ENTITY_HEX:
+ if (b == ';') {
+ entityCount++;
+ state = TEXT;
+ } else if (isHexDigit(b)
+ && (i - entityStart) <= MAX_ENTITY_BODY_LEN) {
+ // continue
+ } else {
+ for (int k = entityStart; k < i; k++) {
+ dst[w++] = src[k];
+ }
+ if (b == '<') {
+ state = LT;
+ } else if (b == '&') {
+ entityStart = i;
+ state = ENTITY;
+ } else {
+ dst[w++] = b;
+ state = TEXT;
+ }
+ }
+ break;
+
case TAG_NAME:
if (isTagNameTerminator(b)) {
int nameLen = i - nameStart;
@@ -332,7 +508,15 @@ public final class HtmlByteStripper {
}
}
- return new Result(w - dstOffset, tagCount);
+ // Unterminated entity at EOF: emit consumed prefix as literal text.
+ if (state == ENTITY || state == ENTITY_NAME || state == ENTITY_NUM
+ || state == ENTITY_DEC || state == ENTITY_HEX) {
+ for (int k = entityStart; k < end; k++) {
+ dst[w++] = src[k];
+ }
+ }
+
+ return new Result(w - dstOffset, tagCount, entityCount);
}
/**
@@ -350,6 +534,16 @@ public final class HtmlByteStripper {
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');
}
+ private static boolean isAsciiDigit(byte b) {
+ int c = b & 0xFF;
+ return c >= '0' && c <= '9';
+ }
+
+ private static boolean isHexDigit(byte b) {
+ int c = b & 0xFF;
+ return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f');
+ }
+
private static boolean isTagNameTerminator(byte b) {
return b == ' ' || b == '\t' || b == '\n' || b == '\r' || b == '>' || b == '/';
}
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
index d540bf2e22..00254dcd96 100644
--- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
@@ -24,7 +24,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Locale;
-import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -76,7 +75,12 @@ public class MojibusterEncodingDetector implements EncodingDetector {
public static final String DEFAULT_MODEL_RESOURCE =
"/org/apache/tika/ml/chardetect/nb-bigram.bin";
- private static final int MAX_PROBE_BYTES = 4096;
+ // Probe sized by tag-stripped content (16 KB target), capped at 512 KB raw.
+ // Markup-heavy pages whose distinguishing bytes (esp. UTF-8 multi-byte
+ // sequences) sit past a fixed 16 KB raw window would otherwise starve the
+ // structural UTF-8 check and NB scoring. See AdaptiveProbe.
+ private static final int PROBE_CONTENT_TARGET = AdaptiveProbe.DEFAULT_CONTENT_TARGET;
+ private static final int PROBE_RAW_CAP = AdaptiveProbe.DEFAULT_RAW_CAP;
/**
* Minimum number of successfully-parsed well-formed tags required
@@ -94,6 +98,15 @@ public class MojibusterEncodingDetector implements EncodingDetector {
*/
private static final int MIN_TAG_COUNT_TO_USE_STRIP = 1;
+ /**
+ * Minimum HTML entity count to apply the stripper even when no
+ * well-formed tags are present. A single stray {@code &}
+ * mention in plain prose shouldn't trigger the strip path, but
+ * entity-heavy content (HTML-quoted text in a plain-text file,
+ * truncated reads where the leading tag was lost, etc.) should.
+ */
+ private static final int MIN_ENTITY_COUNT_TO_USE_STRIP = 3;
+
/**
* Confidence attached to UTF-32 structural candidates — high but
* sub-1.0 so the ResultType.STRUCTURAL flag carries meaning
@@ -111,18 +124,6 @@ public class MojibusterEncodingDetector implements EncodingDetector {
*/
private static final float UTF8_STRUCTURAL_CONF = 0.95f;
- /**
- * Low-evidence threshold (number of high bytes ≥ 0x80) below
- * which {@link #applyLatinSiblingFallback} fires. Short probes
- * (sparse Latin in HTML, vCard fragments) get non-1252 Latin
- * sibling picks from NB on bias / hash-bucket accidents; when
- * the probe decodes byte-identically under windows-1252 we
- * relabel to windows-1252 — the WHATWG-canonical answer. Above
- * this threshold the model has genuine evidence to discriminate
- * sibling code pages.
- */
- private static final int LATIN_FALLBACK_HIGH_BYTE_THRESHOLD = 5;
-
/** Confidence for the windows-1252 fallback emitted on empty/ASCII probes. */
private static final float FALLBACK_CONFIDENCE = 0.1f;
@@ -134,15 +135,28 @@ public class MojibusterEncodingDetector implements EncodingDetector {
* to drop a high-confidence UTF-8 classification on otherwise-valid
* text and fall through to {@code AutoDetectReader.detect}, which
* raises {@code TikaException} when the chain returns no candidates.
- * 0.5% (1 byte per 200) accommodates "tiny corruption" while still
- * rejecting genuinely-non-UTF-8 streams (which would have many more
- * malformed bytes).
+ * Per-byte error rate governing LONG probes (the absolute cap below is a
+ * floor for short ones). 0.01% (~1 malformed sequence per 10 KB)
+ * accommodates real UTF-8 with a few stray/corrupt bytes (e.g. a 150 KB
+ * page with 4 errors = 0.003%) while still rejecting a win-1252 page
+ * misread as UTF-8 (a 20 KB Western page surfaces ~14 invalid sequences =
+ * 0.07%, 7× over).
*
* <p>TACTICAL: remove or revisit when Mojibuster's UTF-8 grammar
* check is replaced with a probabilistic decoder that returns a
* confidence score directly.</p>
*/
- private static final double UTF8_MALFORMED_TOLERANCE = 0.005;
+ private static final double UTF8_MALFORMED_TOLERANCE = 0.0001;
+
+ /**
+ * Absolute floor on tolerated UTF-8 error events for SHORT probes, where a
+ * rate is meaningless (a 20-byte string with 1 bad byte is 5%). The
+ * effective cap is {@code max(this, probeLen * UTF8_MALFORMED_TOLERANCE)} —
+ * so short probes allow 1, long probes are governed by the rate. (Earlier
+ * this was a hard cap applied at all lengths, which wrongly rejected long,
+ * genuinely-UTF-8 pages carrying a couple of stray bytes.)
+ */
+ private static final int UTF8_MAX_TOLERATED_ERRORS = 1;
/** Windows-1252: the WHATWG-canonical default for unlabeled Western content. */
private static final String WIN1252 = "windows-1252";
@@ -251,9 +265,9 @@ public class MojibusterEncodingDetector implements EncodingDetector {
// When the gate fires and the specialist has a confident
// winner, short-circuit: return a single UTF-16LE/BE
// STRUCTURAL candidate. Stride-1 byte bigrams cannot
- // discriminate UTF-16 reliably (see
- // why-stride1-bigrams-dont-work-for-utf16.md), so we keep
- // UTF-16 out of NB training and delegate to the specialist.
+ // discriminate UTF-16 reliably (CJK in UTF-16 produces byte
+ // pairs that alias common ASCII bigrams), so we keep UTF-16
+ // out of NB training and delegate to the specialist.
boolean utf16Gate = StructuralEncodingRules.has2ByteColumnAsymmetryEvidence(probe);
LOG.trace("mojibuster utf16Gate={}", utf16Gate);
if (utf16Gate) {
@@ -296,8 +310,10 @@ public class MojibusterEncodingDetector implements EncodingDetector {
boolean utf8Tolerated = false;
if (utf8 == StructuralEncodingRules.Utf8Result.NOT_UTF8) {
int errors = StructuralEncodingRules.countUtf8Errors(probe);
- if (errors > 0
- && (double) errors / probe.length <= UTF8_MALFORMED_TOLERANCE) {
+ // Length-aware: absolute floor for short probes, rate for long ones.
+ int maxTolerated = Math.max(UTF8_MAX_TOLERATED_ERRORS,
+ (int) (probe.length * UTF8_MALFORMED_TOLERANCE));
+ if (errors > 0 && errors <= maxTolerated) {
utf8Tolerated = true;
LOG.trace("mojibuster utf8 NOT_UTF8 tolerated: {} error events in {}B ({}%)",
errors, probe.length,
@@ -482,40 +498,54 @@ public class MojibusterEncodingDetector implements EncodingDetector {
}
/**
- * Relabel the top result to windows-1252 when all of the following
- * hold:
- * <ul>
- * <li>top candidate is a non-1252 member of
- * {@link CharsetConfusables#SBCS_LATIN_FAMILY};</li>
- * <li>high-byte count <
- * {@link #LATIN_FALLBACK_HIGH_BYTE_THRESHOLD};</li>
- * <li>the probe decodes byte-identically under the candidate
- * and under windows-1252 — no information is lost by the
- * rewrite.</li>
- * </ul>
- * Rationale: on sparse-Latin probes NB picks sibling code pages
- * (ISO-8859-3, x-MacRoman, IBM850) on bias. windows-1252 is the
- * WHATWG-canonical answer and matches downstream test
- * expectations. Mirrors Mojibuster's LATIN_FALLBACK_WIN1252 rule.
+ * Relabel the top result to windows-1252 when top is a non-1252
+ * member of {@link CharsetConfusables#SBCS_LATIN_FAMILY} and
+ * windows-1252 decodes at least as many Unicode-Letter codepoints
+ * at high-byte positions as the candidate does.
+ *
+ * <p>Rationale: NB has a residual bias toward MacRoman / IBM850 /
+ * IBM852 / ISO-8859-X siblings on Western European text where the
+ * underlying bytes are actually windows-1252. Under the wrong
+ * sibling, the high bytes decode to symbols / punctuation /
+ * unassigned codepoints — not letters. Under the correct
+ * windows-1252, they decode to letters (ä, ö, ü, é, ñ, …). So a
+ * letter-count compare directly distinguishes "this is actually
+ * windows-1252 mis-labeled" from "this is genuinely MacRoman".
+ * A real MacRoman document with bytes like 0x88 (à in MacRoman)
+ * decodes to a letter under MacRoman but a symbol (ˆ) under
+ * windows-1252 — letter compare correctly keeps MacRoman.</p>
+ *
+ * <p>Replaces the prior strict gates ({@code countHighBytes < 5}
+ * AND {@code byteIdenticalOnProbe(top, win-1252)}) which left
+ * ≥ 5-high-byte Western European pages unprotected.</p>
*/
private static List<EncodingResult> applyLatinSiblingFallback(byte[] probe,
List<EncodingResult> ranked) {
if (ranked.isEmpty()) {
return ranked;
}
- if (countHighBytes(probe) >= LATIN_FALLBACK_HIGH_BYTE_THRESHOLD) {
- return ranked;
- }
EncodingResult top = ranked.get(0);
String topName = top.getCharset().name();
if (WIN1252.equals(topName)) {
return ranked;
}
- if (!CharsetConfusables.SBCS_LATIN_FAMILY.contains(topName)) {
+ // Scoped to Western European Latin family only. Central
+ // European (win-1250 / ISO-8859-2 / IBM852), Baltic (win-1257 /
+ // ISO-8859-13), Turkish (win-1254), Maltese (ISO-8859-3),
+ // Romanian (ISO-8859-16) etc. are NOT in scope — those
+ // represent different language regions, and rewriting them to
+ // windows-1252 corrupts genuine non-Western content (the
+ // letter-count compare ties on most of their Latin letters
+ // because Unicode classifies both decodings as Letters,
+ // misleading the rule into a wrong flip).
+ if (!CharsetConfusables.WESTERN_LATIN_FAMILY.contains(topName)) {
return ranked;
}
Charset win1252 = Charset.forName(WIN1252);
- if (!DecodeEquivalence.byteIdenticalOnProbe(probe, top.getCharset(), win1252)) {
+ int winLetters = countHighByteLetters(probe, win1252);
+ int topLetters = countHighByteLetters(probe, top.getCharset());
+ // Tie goes to windows-1252 (WHATWG-canonical default).
+ if (winLetters < topLetters) {
return ranked;
}
List<EncodingResult> out = new java.util.ArrayList<>(ranked.size());
@@ -527,6 +557,65 @@ public class MojibusterEncodingDetector implements EncodingDetector {
return out;
}
+ /**
+ * Decode the probe under {@code cs} and count codepoints that
+ * are Unicode "cased letters" (Lu / Ll / Lt) at codepoints ≥
+ * 0x80. Used by the Latin sibling fallback to compare decoded-
+ * text quality between two candidate SBCS encodings.
+ *
+ * <p>Deliberately excludes a few "letter-ish but typographic"
+ * categories that {@link Character#isLetter(int)} would otherwise
+ * count, because they fooled the rule in earlier evals:</p>
+ * <ul>
+ * <li><b>Modifier letters (Lm)</b>: spacing-modifier letterlike
+ * symbols (ʰ ʷ ˆ ˜ ʻ etc.) that some encodings put at
+ * byte positions where the truthful encoding has a symbol /
+ * punctuation.</li>
+ * <li><b>Ordinal indicators</b>: U+00AA (ª), U+00BA (º),
+ * U+207F (ⁿ), U+2122 (™ — not Ll, included for safety).
+ * MacRoman's 0xBB and 0xBC are ª / º respectively; the
+ * windows-1252 truth for byte 0xBB is » (final punctuation,
+ * not a letter). Without this exclusion, MacRoman's
+ * letter count beats win-1252's on probes where » appears.</li>
+ * <li><b>Other letter (Lo)</b>: covers CJK / Korean letterlike
+ * codepoints that occasionally fall out of byte-level
+ * decodes; counting those as "Latin letters" would mislead
+ * the Latin-sibling comparison.</li>
+ * </ul>
+ */
+ private static int countHighByteLetters(byte[] probe, Charset cs) {
+ String decoded;
+ try {
+ decoded = new String(probe, cs);
+ } catch (Exception e) {
+ return 0;
+ }
+ int count = 0;
+ for (int i = 0; i < decoded.length(); ) {
+ int cp = decoded.codePointAt(i);
+ if (cp >= 0x80 && isCasedLatinishLetter(cp)) {
+ count++;
+ }
+ i += Character.charCount(cp);
+ }
+ return count;
+ }
+
+ /**
+ * Returns true for codepoints in Unicode's "cased letter"
+ * categories (Lu / Ll / Lt) but EXCLUDING specific letterlike
+ * typographic symbols (ª, º, ⁿ). See {@link #countHighByteLetters}.
+ */
+ private static boolean isCasedLatinishLetter(int cp) {
+ if (cp == 0x00AA || cp == 0x00BA || cp == 0x207F) {
+ return false; // ª, º, ⁿ — ordinal / superscript indicators
+ }
+ int type = Character.getType(cp);
+ return type == Character.UPPERCASE_LETTER
+ || type == Character.LOWERCASE_LETTER
+ || type == Character.TITLECASE_LETTER;
+ }
+
private static int countHighBytes(byte[] probe) {
int n = 0;
for (byte b : probe) {
@@ -597,10 +686,11 @@ public class MojibusterEncodingDetector implements EncodingDetector {
byte[] dst = new byte[probe.length];
HtmlByteStripper.Result stripped =
HtmlByteStripper.strip(probe, 0, probe.length, dst, 0);
- if (stripped.tagCount < MIN_TAG_COUNT_TO_USE_STRIP) {
- // No well-formed tags found — probe isn't markup (or the
- // bytes don't parse as markup in any ASCII-compatible
- // reading). Use original.
+ if (stripped.tagCount < MIN_TAG_COUNT_TO_USE_STRIP
+ && stripped.entityCount < MIN_ENTITY_COUNT_TO_USE_STRIP) {
+ // No well-formed tags AND not enough entities to be markup —
+ // probe isn't markup (or the bytes don't parse as markup in
+ // any ASCII-compatible reading). Use original.
return probe;
}
byte[] trimmed = new byte[stripped.length];
@@ -622,18 +712,6 @@ public class MojibusterEncodingDetector implements EncodingDetector {
}
private static byte[] readProbe(TikaInputStream tis) throws IOException {
- tis.mark(MAX_PROBE_BYTES);
- byte[] buf = new byte[MAX_PROBE_BYTES];
- try {
- int n = IOUtils.read(tis, buf);
- if (n < buf.length) {
- byte[] trimmed = new byte[n];
- System.arraycopy(buf, 0, trimmed, 0, n);
- return trimmed;
- }
- return buf;
- } finally {
- tis.reset();
- }
+ return AdaptiveProbe.read(tis, PROBE_CONTENT_TARGET, PROBE_RAW_CAP);
}
}
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/NaiveBayesBigramEncodingDetector.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/NaiveBayesBigramEncodingDetector.java
index f0bff4c870..2460656f0c 100644
--- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/NaiveBayesBigramEncodingDetector.java
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/NaiveBayesBigramEncodingDetector.java
@@ -77,20 +77,77 @@ public class NaiveBayesBigramEncodingDetector implements EncodingDetector {
private static final int DEFAULT_TOP_K = 5;
/**
- * Minimum softmax confidence for a candidate to be emitted. When
- * NB is very confident (e.g., top = 0.93 on a long clean EBCDIC
- * probe), the lower-ranked candidates' softmax values fall to ≤
- * 1e-3 and contribute nothing to downstream arbitration except
- * noise. Low-confidence alternatives also give language-based
- * arbitrators (CharSoup) opportunities to pick cross-script
- * decodings that happen to look like valid letters of another
- * script — documented failure mode: English CP500 bytes decoded
- * as IBM424 produce all-Hebrew letters that CharSoup's language
- * model scores as "clean Hebrew" with high margin, beating
- * IBM500's "English with Latin siblings" fit. Dropping noise
- * candidates removes the opportunity.
+ * Per-scored-bigram log-score margin (in nats) that defines
+ * "model is reliably right" vs "model is genuinely uncertain
+ * between candidates." Derived from a calibration run over the
+ * 158K-sample devtest split:
+ *
+ * <ul>
+ * <li>CORRECT picks have top-1-vs-top-2 margin median ≈
+ * 1.5–2 nats/bg, with p10 ≥ 0.22 nats/bg in every
+ * length bucket.</li>
+ * <li>WRONG picks have margin p90 < 0.10 nats/bg in every
+ * length bucket.</li>
+ * </ul>
+ *
+ * <p>A threshold of 0.20 cleanly separates the two regimes.
+ * Candidates within {@code MARGIN_THRESHOLD_NATS_PER_BIGRAM} of
+ * top-1's score (i.e., the model can't reliably tell them apart
+ * from top-1) are emitted into the candidate pool for downstream
+ * arbitration; candidates further away are dropped.</p>
+ *
+ * <p>Softmax-based confidence is deliberately not used here:
+ * softmax saturates to 1.0 on essentially every probe regardless
+ * of how uncertain the model actually is, so it cannot serve as
+ * a candidate-emission gate.</p>
*/
- private static final double MIN_EMIT_CONFIDENCE = 0.01;
+ public static final double MARGIN_THRESHOLD_NATS_PER_BIGRAM = 0.20;
+
+ /**
+ * Per-bigram cross-class total-contribution cap (Type C clipping).
+ * For each distinct bigram in the probe, the top-scoring class's
+ * total contribution (count × logP × idf, after dequantization) is
+ * capped at the runner-up class's contribution + this many nats.
+ *
+ * <p>Defends against corpus-skew pathologies where one class
+ * accumulates extreme bigram mass that swings classification on
+ * one or two byte-pairs alone (e.g., Czech "ČR" digraph in
+ * ISO-8859-2 contributing +186 nats over win-1252 on Italian text).
+ * Length-invariant by construction: the cap is on per-bigram
+ * advantage, regardless of how many times the bigram appears.</p>
+ *
+ * <p>20 nats = e^20 ≈ 5×10^8 probability-ratio advantage per
+ * bigram — preserves legitimate CJK-vs-Latin and other cross-script
+ * signal while bounding the diffuse-corpus-skew tail.</p>
+ */
+ public static final double CAP_PER_BIGRAM_NATS = 20.0;
+
+ /**
+ * Minimum distinct bigrams required before the per-bigram cap
+ * applies. On short probes, each bigram carries proportionally
+ * more signal — clipping would destroy more discrimination than
+ * it saves.
+ */
+ public static final int MIN_DISTINCT_FOR_CAP = 30;
+
+ /**
+ * Minimum distinct-bigram fraction of total-scored-bigrams. Below
+ * this, the input is treated as degenerate (looped / repeated /
+ * corrupt) and {@link #scoreClassesAndCount(byte[])} returns
+ * {@code null} so callers can fall back. Defends against pathological
+ * inputs like {@code "thththth..."} where one bigram appears
+ * hundreds of times.
+ */
+ public static final double MIN_DIVERSITY_RATIO = 0.02;
+
+ /**
+ * Minimum scored bigrams required before the diversity gate
+ * applies. Short probes legitimately have lower diversity ratios
+ * (fewer total bigrams = fewer opportunities for distinct ones)
+ * and shouldn't be gated as degenerate. Above this floor, the
+ * ratio measurement is meaningful.
+ */
+ public static final int MIN_BIGRAMS_FOR_DIVERSITY_GATE = 100;
private final String[] labels;
/** Charset objects cached at load — one {@code Charset.forName} per class, ever. */
@@ -211,62 +268,380 @@ public class NaiveBayesBigramEncodingDetector implements EncodingDetector {
return detect(readProbe(tis));
}
+ /** ASCII whitespace: TAB, LF, VT, FF, CR, SPACE. */
+ private static boolean isWhitespace(int b) {
+ return b == 0x09 || b == 0x0a || b == 0x0b || b == 0x0c
+ || b == 0x0d || b == 0x20;
+ }
+
public List<EncodingResult> detect(byte[] probe) {
- if (probe == null || probe.length < 2) {
+ ScoreResult sr = scoreClassesAndCount(probe);
+ if (sr == null) {
return Collections.emptyList();
}
+ return emitCandidates(sr.scores, sr.scoredBigrams);
+ }
+
+ /**
+ * Score result returned by {@link #scoreClassesAndCount(byte[])}.
+ * Exposes the raw per-class score vector together with the number
+ * of bigrams that actually contributed to the dot product (i.e.,
+ * bigrams with non-zero IDF and not skipped by the whitespace-pair
+ * rule) and the total bigrams in the scored region of the probe.
+ * {@code scoredBigrams} is the unit of "evidence available to NB"
+ * — robust to HTML / whitespace noise in the input because those
+ * bigrams have IDF == 0 and don't contribute.
+ */
+ public static final class ScoreResult {
+ public final double[] scores;
+ public final int scoredBigrams;
+ public final int totalBigrams;
+ public ScoreResult(double[] scores, int scoredBigrams, int totalBigrams) {
+ this.scores = scores;
+ this.scoredBigrams = scoredBigrams;
+ this.totalBigrams = totalBigrams;
+ }
+ }
+
+ /**
+ * Compute the raw per-class score vector for a probe, without
+ * top-K extraction or softmax. Returns {@code null} for null /
+ * tiny probes that can't be scored.
+ */
+ public double[] scoreClasses(byte[] probe) {
+ ScoreResult sr = scoreClassesAndCount(probe);
+ return sr == null ? null : sr.scores;
+ }
+
+ /**
+ * Per-bigram contribution to the per-class score, used for
+ * diagnostic tools that want to understand why a probe scores
+ * one class over another. Returned by
+ * {@link #analyzeBigrams(byte[], int, int)}.
+ */
+ public static final class BigramContrib {
+ public final int bigram; // (b0 << 8) | b1
+ public final double contribA; // logP_A * idf in nats
+ public final double contribB;
+ public BigramContrib(int bigram, double a, double b) {
+ this.bigram = bigram;
+ this.contribA = a;
+ this.contribB = b;
+ }
+ public double diff() {
+ return contribA - contribB;
+ }
+ }
+
+ /**
+ * For each scored bigram in the probe (same skip rules as
+ * {@link #scoreClasses(byte[])}), compute and return its
+ * dequantized contribution to two specified classes' scores.
+ * The list is in probe order, with duplicates allowed (a bigram
+ * that appears N times in the probe yields N entries).
+ */
+ public List<BigramContrib> analyzeBigrams(byte[] probe, int classA, int classB) {
+ List<BigramContrib> out = new java.util.ArrayList<>();
+ if (probe == null || probe.length < 2) {
+ return out;
+ }
+ int len = Math.min(probe.length, MAX_PROBE_BYTES);
+ // perClassDequant[c] folds scale[c] × idfScale already, so
+ // contribution(bigram, c) = logP8[..c] * idf8[bigram] * perClassDequant[c]
+ double dqA = perClassDequant[classA];
+ double dqB = perClassDequant[classB];
+ for (int i = 0; i + 1 < len; i++) {
+ int b0 = probe[i] & 0xFF;
+ int b1 = probe[i + 1] & 0xFF;
+ if (isWhitespace(b0) && isWhitespace(b1)) {
+ continue;
+ }
+ int bigram = (b0 << 8) | b1;
+ int w = idf8[bigram];
+ if (w == 0) {
+ continue;
+ }
+ int base = bigram * numClasses;
+ double contribA = logP8[base + classA] * w * dqA;
+ double contribB = logP8[base + classB] * w * dqB;
+ out.add(new BigramContrib(bigram, contribA, contribB));
+ }
+ return out;
+ }
+
+ /**
+ * Like {@link #scoreClasses(byte[])} but also reports the number
+ * of bigrams that contributed to the dot product vs the total
+ * scored region. Used by offline calibration to bucket samples
+ * by "evidence available" rather than raw byte length.
+ */
+ public ScoreResult scoreClassesAndCount(byte[] probe) {
+ if (probe == null || probe.length < 2) {
+ return null;
+ }
int len = Math.min(probe.length, MAX_PROBE_BYTES);
- // Integer hot loop — CharSoup-style. int8 logP × int8 IDF →
- // int16 product, accumulated into int32 per class. Overflow
- // safety: at MAX_PROBE_BYTES=1024, max 1023 bigrams × 127 × 127
- // ≈ 16.5M per class, well inside int32's 2.1B headroom.
- int[] dots = new int[numClasses];
+ // Pass 1: count distinct bigrams. Whitespace and zero-IDF
+ // bigrams are skipped as in the original hot loop. Counts are
+ // held in a sparse open-addressing int→int hash (see
+ // {@link BigramCountMap}) so per-call working state is
+ // proportional to distinct bigrams (typically a few hundred to
+ // a few thousand) rather than the dense 128 KB
+ // {@code short[65536]} the earlier inner loop used. Iteration
+ // for pass 2 walks the hash's occupied slots directly — no
+ // parallel distinct-bigram array.
+ BigramCountMap counts = new BigramCountMap();
+ int scored = 0;
+ int total = 0;
for (int i = 0; i + 1 < len; i++) {
- int bigram = ((probe[i] & 0xFF) << 8) | (probe[i + 1] & 0xFF);
- int w = idf8[bigram]; // non-negative, 0..127
+ int b0 = probe[i] & 0xFF;
+ int b1 = probe[i + 1] & 0xFF;
+ total++;
+ if (isWhitespace(b0) && isWhitespace(b1)) {
+ continue;
+ }
+ int bigram = (b0 << 8) | b1;
+ int w = idf8[bigram];
if (w == 0) {
- continue; // bigram appears in every class; no signal
+ continue;
+ }
+ scored++;
+ counts.increment(bigram);
+ }
+ int distinctIdx = counts.size();
+
+ // Type A — diversity gate. If the input has too few distinct
+ // bigrams relative to total scored bigrams, it's a degenerate
+ // / looped input ("thththth..." or worse). Abstain — caller
+ // falls back. Only applied above a minimum scored-bigrams
+ // floor, since short probes legitimately have lower diversity
+ // ratios.
+ if (scored >= MIN_BIGRAMS_FOR_DIVERSITY_GATE
+ && (double) distinctIdx / scored < MIN_DIVERSITY_RATIO) {
+ return null;
+ }
+
+ // Type C — per-bigram total-contribution cap. Only applies
+ // when we have enough distinct bigrams that capping any single
+ // one won't destroy a large fraction of the discriminative
+ // signal. Below the floor, short-probe semantics rule: every
+ // bigram counts fully.
+ boolean applyCap = distinctIdx >= MIN_DISTINCT_FOR_CAP;
+
+ // Pass 2: per distinct bigram, compute per-class total
+ // contribution and (when above floor) apply Type C cap.
+ // Order-independent — see analyzeBigrams() for the probe-order
+ // diagnostic path.
+ double[] score = new double[numClasses];
+ double[] contributions = new double[numClasses];
+ int hashCap = counts.capacity();
+ for (int slot = 0; slot < hashCap; slot++) {
+ int bigram = counts.keyAt(slot);
+ if (bigram == -1) {
+ continue;
}
+ int n = counts.countAt(slot);
+ int w = idf8[bigram];
+ double countTimesIdf = (double) n * w;
int base = bigram * numClasses;
+
+ if (!applyCap) {
+ // Fast path: no cap, just accumulate.
+ for (int c = 0; c < numClasses; c++) {
+ score[c] += logP8[base + c] * countTimesIdf * perClassDequant[c];
+ }
+ continue;
+ }
+
+ // logPs are negative; "best" class for the bigram = highest
+ // (least negative) contribution after dequant.
+ double max = Double.NEGATIVE_INFINITY;
+ double secondMax = Double.NEGATIVE_INFINITY;
for (int c = 0; c < numClasses; c++) {
- dots[c] += logP8[base + c] * w;
+ double contrib = logP8[base + c] * countTimesIdf * perClassDequant[c];
+ contributions[c] = contrib;
+ if (contrib > max) {
+ secondMax = max;
+ max = contrib;
+ } else if (contrib > secondMax) {
+ secondMax = contrib;
+ }
+ }
+ // Cap any class whose contribution exceeds runner-up + cap.
+ double capValue = secondMax + CAP_PER_BIGRAM_NATS;
+ if (max > capValue) {
+ for (int c = 0; c < numClasses; c++) {
+ if (contributions[c] > capValue) {
+ contributions[c] = capValue;
+ }
+ }
+ }
+ for (int c = 0; c < numClasses; c++) {
+ score[c] += contributions[c];
}
}
+ return new ScoreResult(score, scored, total);
+ }
- // Single per-class dequantization at end of probe. The
- // perClassDequant constant folds scale[c] × idfScale ×
- // (1/logVocabSize[c]) into one float — the B-3 per-class
- // score normalization comes for free.
- double[] score = new double[numClasses];
- for (int c = 0; c < numClasses; c++) {
- score[c] = dots[c] * perClassDequant[c];
+ /**
+ * Open-addressing {@code int → int} hash map specialised for
+ * counting bigram occurrences during a single
+ * {@link #scoreClassesAndCount(byte[])} pass. Linear probing;
+ * capacity is a power of two; {@code -1} sentinel for empty slots
+ * (bigrams are non-negative 16-bit values so {@code -1} is
+ * unambiguous).
+ *
+ * <p>Per-call local; not thread-safe. Replaces a dense
+ * {@code short[65536]} (128 KB) count array. Memory scales with
+ * actual distinct bigrams in the probe — typically a few hundred
+ * for short probes, a few thousand for diverse probes at the
+ * {@link #MAX_PROBE_BYTES} cap. Initial capacity sized so short
+ * probes never resize; longer probes trigger one or two
+ * power-of-two doublings.
+ */
+ private static final class BigramCountMap {
+
+ /** Initial capacity. 1024 entries × 2 × 4 bytes = 8 KB. */
+ private static final int INITIAL_CAP = 1024;
+ /** Knuth multiplicative hash constant (golden ratio, 32-bit). */
+ private static final int HASH_MULT = 0x9E3779B9;
+
+ private int[] keys;
+ private int[] counts;
+ private int cap;
+ private int mask;
+ /**
+ * Right-shift amount for the multiplicative hash: produces the
+ * top {@code log2(cap)} bits of the multiplied value. Equal
+ * to {@code Integer.numberOfLeadingZeros(mask)} for a
+ * power-of-two capacity.
+ */
+ private int shift;
+ /** Resize when {@code size > threshold}; 50% load factor for fast probing. */
+ private int threshold;
+ private int size;
+
+ BigramCountMap() {
+ this.cap = INITIAL_CAP;
+ this.mask = cap - 1;
+ this.shift = Integer.numberOfLeadingZeros(mask);
+ this.threshold = cap >>> 1;
+ this.keys = new int[cap];
+ this.counts = new int[cap];
+ Arrays.fill(this.keys, -1);
}
- return topK(score, DEFAULT_TOP_K);
+ /** Insert a new bigram or increment the count of an existing one. */
+ void increment(int bigram) {
+ int slot = (bigram * HASH_MULT) >>> shift;
+ while (keys[slot] != -1 && keys[slot] != bigram) {
+ slot = (slot + 1) & mask;
+ }
+ if (keys[slot] == -1) {
+ keys[slot] = bigram;
+ counts[slot] = 1;
+ size++;
+ if (size > threshold) {
+ resize();
+ }
+ } else {
+ counts[slot]++;
+ }
+ }
+
+ /** Number of distinct bigrams stored. */
+ int size() {
+ return size;
+ }
+
+ /** Slot count. Walk slots {@code [0, capacity)} to enumerate entries. */
+ int capacity() {
+ return cap;
+ }
+
+ /** Bigram at {@code slot}, or {@code -1} if the slot is empty. */
+ int keyAt(int slot) {
+ return keys[slot];
+ }
+
+ /** Count at {@code slot}. Undefined for empty slots. */
+ int countAt(int slot) {
+ return counts[slot];
+ }
+
+ private void resize() {
+ int oldCap = cap;
+ int[] oldKeys = keys;
+ int[] oldCounts = counts;
+ cap = oldCap << 1;
+ mask = cap - 1;
+ shift = Integer.numberOfLeadingZeros(mask);
+ threshold = cap >>> 1;
+ keys = new int[cap];
+ counts = new int[cap];
+ Arrays.fill(keys, -1);
+ for (int i = 0; i < oldCap; i++) {
+ int k = oldKeys[i];
+ if (k == -1) {
+ continue;
+ }
+ int slot = (k * HASH_MULT) >>> shift;
+ while (keys[slot] != -1) {
+ slot = (slot + 1) & mask;
+ }
+ keys[slot] = k;
+ counts[slot] = oldCounts[i];
+ }
+ }
+ }
+
+ public String[] getLabels() {
+ return labels.clone();
+ }
+
+ public Charset[] getCharsets() {
+ return charsets.clone();
}
/**
- * Bounded top-K extraction via insertion sort on a size-K primitive
- * array. Avoids {@code Integer[]} boxing + comparator callbacks of
- * {@code Arrays.sort} with a comparator. O(N·K) comparisons total;
- * for K=5, N=35 that's < 180 comparisons, comparable to an O(N
- * log N) sort but with zero allocation beyond the K-sized buffers.
+ * Margin-gated candidate emission. Always emits top-1. Additional
+ * candidates are emitted only when their score is within
+ * {@link #MARGIN_THRESHOLD_NATS_PER_BIGRAM} × {@code scoredBigrams}
+ * of top-1 — i.e., when the model is genuinely close between
+ * top-1 and the alternative. Cap at {@link #DEFAULT_TOP_K}
+ * candidates total.
*
- * <p>Confidence is softmax over the top-K log-likelihoods only —
- * 5 exp() calls instead of numClasses.</p>
+ * <p>The emitted {@code confidence} value is NOT softmax (which
+ * saturates to 1.0 on essentially every probe regardless of true
+ * uncertainty — see {@code feedback_no_softmax_for_ood.md}).
+ * It's a linear margin distance:</p>
+ * <ul>
+ * <li>top-1: 1.0</li>
+ * <li>rank {@code i > 0}:
+ * {@code 1.0 - (top1_score − this_score) / margin_threshold},
+ * clamped to [0.0, 1.0]</li>
+ * </ul>
+ * <p>A candidate at exactly the margin threshold gets confidence
+ * 0.0 and isn't emitted; one at half the threshold gets 0.5;
+ * top-1 always gets 1.0.</p>
*/
- private List<EncodingResult> topK(double[] score, int k) {
- k = Math.min(k, numClasses);
+ private List<EncodingResult> emitCandidates(double[] score, int scoredBigrams) {
+ if (scoredBigrams <= 0) {
+ // No evidence at all — emit nothing. Higher-level callers
+ // (MojibusterEncodingDetector) have their own pure-ASCII /
+ // empty-probe fallbacks.
+ return Collections.emptyList();
+ }
+ double marginThreshold = MARGIN_THRESHOLD_NATS_PER_BIGRAM * scoredBigrams;
+
+ int k = Math.min(DEFAULT_TOP_K, numClasses);
int[] idx = new int[k];
double[] val = new double[k];
Arrays.fill(idx, -1);
Arrays.fill(val, Double.NEGATIVE_INFINITY);
-
for (int c = 0; c < numClasses; c++) {
double s = score[c];
if (s > val[k - 1]) {
- // Shift-right insertion into sorted-desc buffer.
int pos = k - 1;
while (pos > 0 && val[pos - 1] < s) {
val[pos] = val[pos - 1];
@@ -277,35 +652,26 @@ public class NaiveBayesBigramEncodingDetector implements EncodingDetector {
idx[pos] = c;
}
}
+ if (idx[0] < 0) {
+ return Collections.emptyList();
+ }
+ double top1Score = val[0];
- // Softmax over top-K only.
- double maxScore = val[0];
- double sumExp = 0.0;
- double[] expBuf = new double[k];
- int filled = 0;
+ List<EncodingResult> out = new ArrayList<>(k);
for (int i = 0; i < k; i++) {
if (idx[i] < 0) {
break;
}
- expBuf[i] = Math.exp(val[i] - maxScore);
- sumExp += expBuf[i];
- filled = i + 1;
- }
-
- List<EncodingResult> out = new ArrayList<>(filled);
- for (int i = 0; i < filled; i++) {
Charset cs = charsets[idx[i]];
if (cs == null) {
- continue; // training-only label with no Java charset
+ continue;
}
- double conf = expBuf[i] / sumExp;
- // Always emit top-1 (even if tiny — at least one result
- // keeps the pipeline from going empty). For the rest,
- // drop below MIN_EMIT_CONFIDENCE: those are noise and
- // cause downstream arbiters to pick cross-script decodings.
- if (i > 0 && conf < MIN_EMIT_CONFIDENCE) {
+ double gap = top1Score - val[i];
+ if (i > 0 && gap >= marginThreshold) {
break;
}
+ double conf = (i == 0) ? 1.0
+ : Math.max(0.0, 1.0 - gap / marginThreshold);
out.add(new EncodingResult(cs, (float) conf,
labels[idx[i]], EncodingResult.ResultType.STATISTICAL));
}
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/nb-bigram.bin b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/nb-bigram.bin
index 505563f1c6..bcfce41d67 100644
Binary files a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/nb-bigram.bin and b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/nb-bigram.bin differ
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/AdaptiveProbeTest.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/AdaptiveProbeTest.java
new file mode 100644
index 0000000000..44c97d8c92
--- /dev/null
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/AdaptiveProbeTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.chardetect;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.io.TikaInputStream;
+
+public class AdaptiveProbeTest {
+
+ private static byte[] probe(byte[] raw, int contentTarget, int rawCap) throws IOException {
+ try (TikaInputStream tis = TikaInputStream.get(raw)) {
+ return AdaptiveProbe.read(tis, contentTarget, rawCap);
+ }
+ }
+
+ /** Plain text below the content target: read everything, stop at EOF. */
+ @Test
+ public void shortPlainTextReadsToEof() throws Exception {
+ byte[] raw = "hello world, this is plain text".getBytes(StandardCharsets.UTF_8);
+ byte[] p = probe(raw, 16384, 524288);
+ assertEquals(raw.length, p.length);
+ }
+
+ /** No tags: content == raw, so the read stops at the content target. */
+ @Test
+ public void plainTextStopsAtContentTarget() throws Exception {
+ byte[] raw = new byte[200_000];
+ java.util.Arrays.fill(raw, (byte) 'a');
+ byte[] p = probe(raw, 16384, 524288);
+ // Stops within one chunk past the target (chunked by contentTarget).
+ assertTrue(p.length >= 16384 && p.length <= 32768,
+ "expected ~content target, got " + p.length);
+ }
+
+ /** Markup-heavy lead: must read past 16 KB raw to accumulate body text. */
+ @Test
+ public void markupHeavyReadsPastFixedWindow() throws Exception {
+ StringBuilder sb = new StringBuilder();
+ // ~40 KB of tags yielding almost no text, then real body content.
+ for (int i = 0; i < 4000; i++) {
+ sb.append("<div class=\"x\"></div>");
+ }
+ int markupBytes = sb.length();
+ for (int i = 0; i < 20000; i++) {
+ sb.append("content ");
+ }
+ byte[] raw = sb.toString().getBytes(StandardCharsets.UTF_8);
+ byte[] p = probe(raw, 16384, 524288);
+ assertTrue(p.length > markupBytes,
+ "should read past the markup block (" + markupBytes + "), got " + p.length);
+ }
+
+ /** All markup, no body: bounded by the raw cap. */
+ @Test
+ public void allMarkupBoundedByRawCap() throws Exception {
+ StringBuilder sb = new StringBuilder();
+ while (sb.length() < 300_000) {
+ sb.append("<a href=\"#\"></a>");
+ }
+ byte[] raw = sb.toString().getBytes(StandardCharsets.UTF_8);
+ int rawCap = 65536;
+ byte[] p = probe(raw, 16384, rawCap);
+ assertEquals(rawCap, p.length);
+ }
+
+ /** Multi-byte text with no ASCII tags stops at the content target (no over-read). */
+ @Test
+ public void utf16NoTagsStopsAtContentTarget() throws Exception {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < 100_000; i++) {
+ sb.append('中'); // CJK; no '<' so tagCount stays 0
+ }
+ byte[] raw = sb.toString().getBytes(StandardCharsets.UTF_16LE);
+ byte[] p = probe(raw, 16384, 524288);
+ assertTrue(p.length >= 16384 && p.length <= 32768,
+ "no tags -> content==raw -> stop near target, got " + p.length);
+ }
+
+ /** mark/reset must leave the stream fully re-readable. */
+ @Test
+ public void streamIsResetAfterProbe() throws Exception {
+ byte[] raw = new byte[100_000];
+ java.util.Arrays.fill(raw, (byte) 'z');
+ try (TikaInputStream tis = TikaInputStream.get(raw)) {
+ AdaptiveProbe.read(tis, 16384, 524288);
+ byte[] all = tis.readAllBytes();
+ assertEquals(raw.length, all.length);
+ }
+ }
+
+ /** Empty input returns an empty array, never null. */
+ @Test
+ public void emptyInputReturnsEmpty() throws Exception {
+ byte[] p = probe(new byte[0], 16384, 524288);
+ assertEquals(0, p.length);
+ }
+}
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/CalibrateTopK.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/CalibrateTopK.java
new file mode 100644
index 0000000000..cbd9b1172c
--- /dev/null
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/CalibrateTopK.java
@@ -0,0 +1,353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.chardetect;
+
+import java.io.DataInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.zip.GZIPInputStream;
+
+/**
+ * Calibration tool — for each devtest sample, run the bigram NB and
+ * record where the true label appears in the ranked candidate list and
+ * what softmax confidence it carries. Bucket by scored-bigram count
+ * (number of bigrams that actually contributed to the dot product,
+ * which is the right unit of "evidence available to NB" regardless of
+ * raw input length and HTML noise).
+ *
+ * <p>Outputs:</p>
+ * <ul>
+ * <li>For each scored-count bucket: top-1 accuracy, top-3 / top-5 /
+ * top-10 cumulative coverage, MIN-confidence at which 95% / 99%
+ * coverage is achieved.</li>
+ * <li>Distribution of scored-bigram counts across devtest.</li>
+ * <li>Optionally: spot-check specific probe files to locate them
+ * on the (scored-count, top-1-margin, true-label-rank) plane.</li>
+ * </ul>
+ */
+public final class CalibrateTopK {
+
+ private CalibrateTopK() {
+ }
+
+ private static final int[] BUCKETS = {
+ 0, 50, 100, 200, 400, 800, 1600, 3200, 6400, 12000, 16000
+ };
+
+ public static void main(String[] args) throws IOException {
+ Path devtestDir = null;
+ Path modelPath = null;
+ Path probeDir = null;
+ String probesArg = null;
+ int maxSamplesPerClass = 5_000;
+ for (int i = 0; i < args.length; i++) {
+ switch (args[i]) {
+ case "--devtest":
+ devtestDir = Paths.get(args[++i].replaceFirst("^~",
+ System.getProperty("user.home")));
+ break;
+ case "--model":
+ modelPath = Paths.get(args[++i]);
+ break;
+ case "--probes":
+ probesArg = args[++i];
+ break;
+ case "--probe-dir":
+ probeDir = Paths.get(args[++i].replaceFirst("^~",
+ System.getProperty("user.home")));
+ break;
+ case "--max-samples-per-class":
+ maxSamplesPerClass = Integer.parseInt(args[++i]);
+ break;
+ default:
+ System.err.println("Unknown arg: " + args[i]);
+ System.exit(1);
+ }
+ }
+ if (modelPath == null) {
+ System.err.println("Usage: CalibrateTopK --model <bin>"
+ + " [--devtest <dir>] [--probe-dir <dir> --probes c,...,c]"
+ + " [--max-samples-per-class N]");
+ System.exit(1);
+ }
+ NaiveBayesBigramEncodingDetector det;
+ try (InputStream is = Files.newInputStream(modelPath)) {
+ det = new NaiveBayesBigramEncodingDetector(is);
+ }
+ String[] labels = det.getLabels();
+ Map<String, Integer> labelIdx = new HashMap<>();
+ for (int i = 0; i < labels.length; i++) {
+ labelIdx.put(labels[i], i);
+ }
+
+ if (devtestDir != null) {
+ runDevtest(det, labels, labelIdx, devtestDir, maxSamplesPerClass);
+ }
+
+ if (probesArg != null && probeDir != null) {
+ String[] ids = probesArg.split(",");
+ System.out.println();
+ System.out.println("=== Per-probe spot check ===");
+ for (String id : ids) {
+ Path p = probeDir.resolve(id);
+ if (!Files.exists(p)) {
+ System.err.println("Missing probe: " + p);
+ continue;
+ }
+ byte[] bytes = Files.readAllBytes(p);
+ NaiveBayesBigramEncodingDetector.ScoreResult sr =
+ det.scoreClassesAndCount(bytes);
+ if (sr == null) {
+ System.out.println(id + " (no score)");
+ continue;
+ }
+ Rank r = rank(sr.scores, labels);
+ int win1252 = labelIdx.getOrDefault("windows-1252", -1);
+ int win1252Rank = win1252 >= 0 ? r.rankOf(win1252) : -1;
+ double sc = Math.max(1, sr.scoredBigrams);
+ double top1NatsPerBg = r.scores[r.idxRanked[0]] / sc;
+ double top2NatsPerBg = r.scores[r.idxRanked[1]] / sc;
+ double marginNatsPerBg = top1NatsPerBg - top2NatsPerBg;
+ double top1Z = r.zOf(r.idxRanked[0]);
+ double winZ = win1252 >= 0 ? r.zOf(win1252) : 0.0;
+ double winNatsPerBg = win1252 >= 0 ? r.scores[win1252] / sc : 0.0;
+ System.out.printf(Locale.ROOT,
+ "%-30s scored=%5d top-1=%-15s nats/bg=%+7.3f z=%+5.2f "
+ + "top2=%-15s margin=%+6.3f win-1252@rank=%2d nats/bg=%+7.3f z=%+5.2f%n",
+ id.substring(id.indexOf('/') + 1, id.indexOf('/') + 13), sr.scoredBigrams,
+ labels[r.idxRanked[0]], top1NatsPerBg, top1Z,
+ labels[r.idxRanked[1]], marginNatsPerBg,
+ win1252Rank, winNatsPerBg, winZ);
+ }
+ }
+ }
+
+ private static void runDevtest(NaiveBayesBigramEncodingDetector det,
+ String[] labels, Map<String, Integer> labelIdx,
+ Path devtestDir, int maxSamplesPerClass) throws IOException {
+ // Per bucket: counts of true rank, per-bigram log-margin distribution.
+ int B = BUCKETS.length;
+ long[][] rankCountByBucket = new long[B][20];
+ long[][] beyondTopByBucket = new long[B][1];
+ long[] sampleCountByBucket = new long[B];
+ // CORRECT picks: per-bigram log-margin (top1 - top2) for correct predictions
+ @SuppressWarnings("unchecked")
+ List<Double>[] correctMarginByBucket = new List[B];
+ for (int i = 0; i < B; i++) correctMarginByBucket[i] = new ArrayList<>();
+ // WRONG picks: per-bigram log-margin for wrong predictions
+ @SuppressWarnings("unchecked")
+ List<Double>[] wrongMarginByBucket = new List[B];
+ for (int i = 0; i < B; i++) wrongMarginByBucket[i] = new ArrayList<>();
+ // ALL: per-bigram score gap between TRUE label and top-1 (for true label rank > 0,
+ // negative; this tells us by how much the model misses on wrong picks)
+ @SuppressWarnings("unchecked")
+ List<Double>[] trueVsTop1MarginByBucket = new List[B];
+ for (int i = 0; i < B; i++) trueVsTop1MarginByBucket[i] = new ArrayList<>();
+
+ List<Path> files;
+ try (Stream<Path> s = Files.list(devtestDir)) {
+ files = s.filter(p -> p.getFileName().toString().endsWith(".bin.gz"))
+ .sorted()
+ .collect(Collectors.toList());
+ }
+ long total = 0;
+ for (Path f : files) {
+ String label = f.getFileName().toString().replaceAll("\\.bin\\.gz$", "");
+ Integer trueIdx = labelIdx.get(label);
+ if (trueIdx == null) continue;
+ int sampled = 0;
+ try (InputStream fis = new FileInputStream(f.toFile());
+ GZIPInputStream gis = new GZIPInputStream(fis);
+ DataInputStream dis = new DataInputStream(gis)) {
+ while (sampled < maxSamplesPerClass) {
+ int len;
+ try {
+ len = dis.readUnsignedShort();
+ } catch (java.io.EOFException eof) {
+ break;
+ }
+ byte[] sample = new byte[len];
+ dis.readFully(sample);
+ NaiveBayesBigramEncodingDetector.ScoreResult sr =
+ det.scoreClassesAndCount(sample);
+ if (sr == null) continue;
+ int bucket = bucketFor(sr.scoredBigrams);
+ sampleCountByBucket[bucket]++;
+ total++;
+ Rank r = rank(sr.scores, labels);
+ int trueRank = r.rankOf(trueIdx);
+ if (trueRank < 20) rankCountByBucket[bucket][trueRank]++;
+ else beyondTopByBucket[bucket][0]++;
+ // Per-bigram log-margin top1 vs top2 (in nats / scored-bigram).
+ // Score is already in nats (log-probability units after dequant).
+ double margin = (sr.scores[r.idxRanked[0]] - sr.scores[r.idxRanked[1]])
+ / Math.max(1, sr.scoredBigrams);
+ if (trueRank == 0) {
+ correctMarginByBucket[bucket].add(margin);
+ } else {
+ wrongMarginByBucket[bucket].add(margin);
+ }
+ // How far the true label is from top-1, per-bigram. Zero
+ // when correct, negative when the model missed.
+ double trueVsTop = (sr.scores[trueIdx] - sr.scores[r.idxRanked[0]])
+ / Math.max(1, sr.scoredBigrams);
+ trueVsTop1MarginByBucket[bucket].add(trueVsTop);
+ sampled++;
+ }
+ }
+ }
+ System.out.printf(Locale.ROOT, "Total devtest samples scored: %,d%n%n", total);
+
+ System.out.println("=== Top-K cumulative coverage by scored-bigram-count bucket ===");
+ System.out.printf(Locale.ROOT, "%-15s %10s %8s %8s %8s %8s %8s %8s%n",
+ "bucket", "samples", "top-1", "top-2", "top-3", "top-5", "top-10", ">=20");
+ for (int b = 0; b < B; b++) {
+ long n = sampleCountByBucket[b];
+ if (n == 0) continue;
+ long c1 = rankCountByBucket[b][0];
+ long c2 = c1 + rankCountByBucket[b][1];
+ long c3 = c2 + rankCountByBucket[b][2];
+ long c5 = c3 + rankCountByBucket[b][3] + rankCountByBucket[b][4];
+ long c10 = c5;
+ for (int k = 5; k < 10; k++) c10 += rankCountByBucket[b][k];
+ long beyond = beyondTopByBucket[b][0];
+ String label = b == B - 1
+ ? String.format(Locale.ROOT, "%d+", BUCKETS[b])
+ : String.format(Locale.ROOT, "%d-%d", BUCKETS[b], BUCKETS[b + 1] - 1);
+ System.out.printf(Locale.ROOT, "%-15s %,10d %7.2f%% %7.2f%% %7.2f%% %7.2f%% %7.2f%% %7.2f%%%n",
+ label, n,
+ 100.0 * c1 / n, 100.0 * c2 / n, 100.0 * c3 / n,
+ 100.0 * c5 / n, 100.0 * c10 / n,
+ 100.0 * beyond / n);
+ }
+
+ System.out.println();
+ System.out.println("=== Per-bigram log-margin (nats/scored-bigram) top-1 vs top-2 ===");
+ System.out.println("(How decisively the model favors top-1 over top-2, normalized by evidence.");
+ System.out.println(" Compare CORRECT-pick distribution vs WRONG-pick distribution per bucket.)");
+ System.out.printf(Locale.ROOT, "%-15s %10s %12s %12s %12s %12s %12s %12s%n",
+ "bucket", "n-correct", "corr-p10", "corr-p50", "corr-p90", "n-wrong", "wrong-p10", "wrong-p90");
+ for (int b = 0; b < B; b++) {
+ long n = sampleCountByBucket[b];
+ if (n == 0) continue;
+ List<Double> corr = correctMarginByBucket[b];
+ List<Double> wrong = wrongMarginByBucket[b];
+ Collections.sort(corr);
+ Collections.sort(wrong);
+ String label = b == B - 1
+ ? String.format(Locale.ROOT, "%d+", BUCKETS[b])
+ : String.format(Locale.ROOT, "%d-%d", BUCKETS[b], BUCKETS[b + 1] - 1);
+ System.out.printf(Locale.ROOT,
+ "%-15s %,10d %12.5f %12.5f %12.5f %,10d %12.5f %12.5f%n",
+ label, (long) corr.size(),
+ percentile(corr, 10), percentile(corr, 50), percentile(corr, 90),
+ (long) wrong.size(),
+ percentile(wrong, 10), percentile(wrong, 90));
+ }
+
+ System.out.println();
+ System.out.println("=== True-label score gap vs top-1 (nats/scored-bigram) ===");
+ System.out.println("(0 when correct. Negative when wrong — magnitude shows how far model missed.");
+ System.out.println(" Lower decile values are the hardest wrong picks per bucket.)");
+ System.out.printf(Locale.ROOT, "%-15s %10s %12s %12s %12s %12s%n",
+ "bucket", "samples", "p1", "p5", "p10", "p50");
+ for (int b = 0; b < B; b++) {
+ long n = sampleCountByBucket[b];
+ if (n == 0) continue;
+ List<Double> diffs = trueVsTop1MarginByBucket[b];
+ Collections.sort(diffs);
+ String label = b == B - 1
+ ? String.format(Locale.ROOT, "%d+", BUCKETS[b])
+ : String.format(Locale.ROOT, "%d-%d", BUCKETS[b], BUCKETS[b + 1] - 1);
+ System.out.printf(Locale.ROOT, "%-15s %,10d %12.5f %12.5f %12.5f %12.5f%n",
+ label, n,
+ percentile(diffs, 1), percentile(diffs, 5),
+ percentile(diffs, 10), percentile(diffs, 50));
+ }
+ }
+
+ private static int bucketFor(int scored) {
+ for (int i = BUCKETS.length - 1; i >= 0; i--) {
+ if (scored >= BUCKETS[i]) return i;
+ }
+ return 0;
+ }
+
+ private static double percentile(List<Double> sorted, double pctile) {
+ if (sorted.isEmpty()) return Double.NaN;
+ int idx = (int) Math.floor(pctile / 100.0 * sorted.size());
+ if (idx < 0) idx = 0;
+ if (idx >= sorted.size()) idx = sorted.size() - 1;
+ return sorted.get(idx);
+ }
+
+ private static final class Rank {
+ final int[] idxRanked; // class index sorted by score desc
+ final double[] scores; // raw scores by class index
+ final double mean; // mean of scores across all classes
+ final double std; // stddev of scores across all classes
+ Rank(int[] idxRanked, double[] scores, double mean, double std) {
+ this.idxRanked = idxRanked;
+ this.scores = scores;
+ this.mean = mean;
+ this.std = std;
+ }
+ int rankOf(int classIdx) {
+ for (int k = 0; k < idxRanked.length; k++) {
+ if (idxRanked[k] == classIdx) return k;
+ }
+ return -1;
+ }
+ /** Z-score of a class's raw score relative to the per-probe
+ * class-score distribution. Top-1 typically gets a large
+ * positive z; classes the model thinks are impossible get
+ * large negative z. */
+ double zOf(int classIdx) {
+ if (std <= 0) return 0.0;
+ return (scores[classIdx] - mean) / std;
+ }
+ }
+
+ private static Rank rank(double[] scores, String[] labels) {
+ int n = scores.length;
+ Integer[] idx = new Integer[n];
+ for (int i = 0; i < n; i++) idx[i] = i;
+ Arrays.sort(idx, (a, b) -> Double.compare(scores[b], scores[a]));
+ double sum = 0;
+ for (double s : scores) sum += s;
+ double mean = sum / n;
+ double sq = 0;
+ for (double s : scores) sq += (s - mean) * (s - mean);
+ double std = Math.sqrt(sq / n);
+ int[] idxRanked = new int[n];
+ for (int k = 0; k < n; k++) idxRanked[k] = idx[k];
+ return new Rank(idxRanked, scores, mean, std);
+ }
+}
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/CheckUtf8OnFile.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/CheckUtf8OnFile.java
new file mode 100644
index 0000000000..927bb52b51
--- /dev/null
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/CheckUtf8OnFile.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.chardetect;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Locale;
+
+/**
+ * Quick-and-dirty: run checkUtf8 on a list of files and report the
+ * result + error count + post-strip result.
+ */
+public final class CheckUtf8OnFile {
+
+ private CheckUtf8OnFile() {
+ }
+
+ public static void main(String[] args) throws Exception {
+ Path probeDir = null;
+ String[] probes = null;
+ for (int i = 0; i < args.length; i++) {
+ switch (args[i]) {
+ case "--probe-dir":
+ probeDir = Paths.get(args[++i].replaceFirst("^~",
+ System.getProperty("user.home")));
+ break;
+ case "--probes":
+ probes = args[++i].split(",");
+ break;
+ default:
+ System.err.println("Unknown arg: " + args[i]);
+ System.exit(1);
+ }
+ }
+ if (probeDir == null || probes == null) {
+ System.err.println("Usage: CheckUtf8OnFile --probe-dir <dir> --probes p1,p2,...");
+ System.exit(1);
+ }
+ for (String pid : probes) {
+ Path p = probeDir.resolve(pid);
+ if (!Files.exists(p)) {
+ System.err.println("Missing: " + p);
+ continue;
+ }
+ byte[] bytes = Files.readAllBytes(p);
+ String shortId = pid.contains("/")
+ ? pid.substring(pid.indexOf('/') + 1, pid.indexOf('/') + 13) : pid;
+
+ StructuralEncodingRules.Utf8Result rawR = StructuralEncodingRules.checkUtf8(bytes);
+ int rawErrors = StructuralEncodingRules.countUtf8Errors(bytes);
+
+ byte[] dst = new byte[bytes.length];
+ HtmlByteStripper.Result sr = HtmlByteStripper.strip(bytes, 0, bytes.length, dst, 0);
+ byte[] stripped = (sr.tagCount >= 1)
+ ? java.util.Arrays.copyOf(dst, sr.length) : bytes;
+ StructuralEncodingRules.Utf8Result strpR = StructuralEncodingRules.checkUtf8(stripped);
+ int strpErrors = StructuralEncodingRules.countUtf8Errors(stripped);
+
+ System.out.printf(Locale.ROOT,
+ "%-14s raw=%6dB result=%-14s errors=%4d (%.4f%%) "
+ + "strip=%6dB result=%-14s errors=%4d (%.4f%%)%n",
+ shortId, bytes.length, rawR, rawErrors,
+ 100.0 * rawErrors / Math.max(1, bytes.length),
+ stripped.length, strpR, strpErrors,
+ 100.0 * strpErrors / Math.max(1, stripped.length));
+ }
+ }
+}
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/HtmlByteStripperTest.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/HtmlByteStripperTest.java
new file mode 100644
index 0000000000..74aa691728
--- /dev/null
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/HtmlByteStripperTest.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.chardetect;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+import org.junit.jupiter.api.Test;
+
+public class HtmlByteStripperTest {
+
+ /** Helper: strip a string and return the (text, tagCount, entityCount) tuple. */
+ private static StripOutcome strip(String input) {
+ byte[] src = input.getBytes(StandardCharsets.US_ASCII);
+ byte[] dst = new byte[src.length];
+ HtmlByteStripper.Result r = HtmlByteStripper.strip(src, 0, src.length, dst, 0);
+ return new StripOutcome(new String(dst, 0, r.length, StandardCharsets.US_ASCII),
+ r.tagCount, r.entityCount);
+ }
+
+ private static final class StripOutcome {
+ final String text;
+ final int tagCount;
+ final int entityCount;
+ StripOutcome(String text, int tagCount, int entityCount) {
+ this.text = text;
+ this.tagCount = tagCount;
+ this.entityCount = entityCount;
+ }
+ }
+
+ /** Helper: tagCount when stripping the given bytes (tags+entities). */
+ private static int tagCount(byte[] src) {
+ byte[] dst = new byte[src.length];
+ return HtmlByteStripper.strip(src, 0, src.length, dst, 0).tagCount;
+ }
+
+ @Test
+ public void multiByteUnicodeIsNotTagStripped() {
+ // The byte-level stripper must not mangle UTF-16/UTF-32: those bytes
+ // don't form single-byte ASCII tags, so tagCount stays 0 and callers
+ // (Mojibuster / JunkFilter) fall back to the raw bytes via their
+ // `tagCount > 0` gate. Regression guard for the "does the byte
+ // stripper botch wide Unicode?" question.
+ String html = "<html><head><title>商品</title></head>"
+ + "<body><p>这是中文测试 with markup</p></body></html>";
+ // ASCII-compatible encodings: tags ARE recognized (and safely stripped).
+ assertTrue(tagCount(html.getBytes(StandardCharsets.UTF_8)) > 0,
+ "UTF-8 tags should be recognized");
+ assertTrue(tagCount(html.getBytes(Charset.forName("GBK"))) > 0,
+ "GBK (ASCII-compatible) tags should be recognized");
+ // Wide Unicode: no single-byte ASCII tags → tagCount 0 → strip not used.
+ assertEquals(0, tagCount(html.getBytes(Charset.forName("UTF-16LE"))),
+ "UTF-16LE must not register tags");
+ assertEquals(0, tagCount(html.getBytes(Charset.forName("UTF-16BE"))),
+ "UTF-16BE must not register tags");
+ assertEquals(0, tagCount(html.getBytes(Charset.forName("UTF-32LE"))),
+ "UTF-32LE must not register tags");
+ assertEquals(0, tagCount(html.getBytes(Charset.forName("UTF-32BE"))),
+ "UTF-32BE must not register tags");
+ }
+
+ @Test
+ public void stripTagsPreservesEntitiesForJunkDetection() {
+ // JunkFilter path: tags removed, entities KEPT (expanded later in
+ // string space). Charset path (default strip) removes both.
+ String in = "<p>Copyright © 2024 café</p>";
+ byte[] src = in.getBytes(StandardCharsets.US_ASCII);
+ byte[] dstA = new byte[src.length];
+ byte[] dstB = new byte[src.length];
+ HtmlByteStripper.Result tagsOnly =
+ HtmlByteStripper.stripTags(src, 0, src.length, dstA, 0);
+ HtmlByteStripper.Result both =
+ HtmlByteStripper.stripTagsAndEntities(src, 0, src.length, dstB, 0);
+ assertEquals("Copyright © 2024 café",
+ new String(dstA, 0, tagsOnly.length, StandardCharsets.US_ASCII));
+ assertEquals("Copyright 2024 caf",
+ new String(dstB, 0, both.length, StandardCharsets.US_ASCII));
+ // tagsOnly does not count entities (it doesn't enter the entity path)
+ assertEquals(0, tagsOnly.entityCount);
+ assertEquals(2, both.entityCount);
+ }
+
+ @Test
+ public void namedEntityIsStripped() {
+ StripOutcome r = strip("hello & world");
+ assertEquals("hello world", r.text);
+ assertEquals(1, r.entityCount);
+ assertEquals(0, r.tagCount);
+ }
+
+ @Test
+ public void decimalNumericEntityIsStripped() {
+ StripOutcome r = strip("foo © bar");
+ assertEquals("foo bar", r.text);
+ assertEquals(1, r.entityCount);
+ }
+
+ @Test
+ public void hexNumericEntityIsStripped() {
+ StripOutcome r = strip("foo © bar");
+ assertEquals("foo bar", r.text);
+ assertEquals(1, r.entityCount);
+ }
+
+ @Test
+ public void hexNumericEntityUppercaseXIsStripped() {
+ StripOutcome r = strip("a©b");
+ assertEquals("ab", r.text);
+ assertEquals(1, r.entityCount);
+ }
+
+ @Test
+ public void ampersandFollowedByLetterWithoutSemicolonIsLiteral() {
+ // AT&T pattern: & followed by letter(s) but no closing ';'
+ StripOutcome r = strip("AT&T Inc");
+ assertEquals("AT&T Inc", r.text);
+ assertEquals(0, r.entityCount);
+ }
+
+ @Test
+ public void ampersandFollowedByNonLetterIsLiteral() {
+ // Q&A pattern: & followed by uppercase letter then space → bailout
+ StripOutcome r = strip("Q&A session");
+ assertEquals("Q&A session", r.text);
+ assertEquals(0, r.entityCount);
+ }
+
+ @Test
+ public void ampersandFollowedBySpaceIsLiteral() {
+ StripOutcome r = strip("a & b");
+ assertEquals("a & b", r.text);
+ assertEquals(0, r.entityCount);
+ }
+
+ @Test
+ public void ampersandAtEndOfInputIsLiteral() {
+ StripOutcome r = strip("end&");
+ assertEquals("end&", r.text);
+ assertEquals(0, r.entityCount);
+ }
+
+ @Test
+ public void unclosedEntityNameAtEndOfInputIsLiteral() {
+ StripOutcome r = strip("end&foo");
+ assertEquals("end&foo", r.text);
+ assertEquals(0, r.entityCount);
+ }
+
+ @Test
+ public void unclosedNumericEntityAtEndOfInputIsLiteral() {
+ StripOutcome r = strip("end{");
+ assertEquals("end{", r.text);
+ assertEquals(0, r.entityCount);
+ }
+
+ @Test
+ public void entityExceedingLengthCapIsLiteral() {
+ // Standard HTML5 entity longer than the 16-byte cap.
+ StripOutcome r = strip("x∳y");
+ // The cap kicks in mid-body; the consumed prefix is emitted as
+ // literal text, then the rest of the bytes follow as text.
+ // Exact prefix depends on cap; key assertion is the entity was
+ // NOT counted as stripped.
+ assertEquals(0, r.entityCount);
+ // The full input is preserved as text (cap bailout emits what
+ // it consumed, and the remaining tail follows naturally).
+ assertEquals("x∳y", r.text);
+ }
+
+ @Test
+ public void adjacentEntitiesAreAllStripped() {
+ StripOutcome r = strip("&&&");
+ assertEquals("", r.text);
+ assertEquals(3, r.entityCount);
+ }
+
+ @Test
+ public void ampersandCascadingIntoTagWorks() {
+ // & followed by letters then '<' should emit the bailout prefix
+ // and then transition into tag-stripping.
+ StripOutcome r = strip("a&foo<b>c");
+ assertEquals("a&fooc", r.text);
+ assertEquals(0, r.entityCount);
+ assertEquals(1, r.tagCount);
+ }
+
+ @Test
+ public void ampersandCascadingIntoAnotherEntity() {
+ // & followed by non-entity content then another & — the
+ // first '&' should emit literal, second '&' starts a new entity.
+ StripOutcome r = strip("a&!&b");
+ assertEquals("a&!b", r.text);
+ assertEquals(1, r.entityCount);
+ }
+
+ @Test
+ public void tagWithEntityInBodyStripsBoth() {
+ StripOutcome r = strip("<p>hello world</p>");
+ assertEquals("helloworld", r.text);
+ assertEquals(2, r.tagCount);
+ assertEquals(1, r.entityCount);
+ }
+
+ @Test
+ public void plainTextNoMarkup() {
+ StripOutcome r = strip("just plain text, no markup at all");
+ assertEquals("just plain text, no markup at all", r.text);
+ assertEquals(0, r.tagCount);
+ assertEquals(0, r.entityCount);
+ }
+
+ @Test
+ public void emptyEntityIsLiteral() {
+ // "&;" — & followed immediately by ';' (not letter / not '#')
+ StripOutcome r = strip("a&;b");
+ assertEquals("a&;b", r.text);
+ assertEquals(0, r.entityCount);
+ }
+
+ @Test
+ public void numericEmptyBodyIsLiteral() {
+ // "&#;" — &# followed by ';' (not 'x' / not digit)
+ StripOutcome r = strip("a&#;b");
+ assertEquals("a&#;b", r.text);
+ assertEquals(0, r.entityCount);
+ }
+}
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/InspectBigramContributions.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/InspectBigramContributions.java
new file mode 100644
index 0000000000..cd04a44eba
--- /dev/null
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/InspectBigramContributions.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.chardetect;
+
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Per-bigram contribution diagnostic. For each probe file, picks a
+ * specific (classA vs classB) comparison and aggregates bigram
+ * contributions by bigram identity. Output: which bigrams in this
+ * probe most push the decision toward classA vs classB, with byte
+ * values, hit count, total contribution, and decoded chars under
+ * each charset.
+ *
+ * <p>Used to investigate why Mojibuster gives GB18030 a slight edge
+ * over windows-1252 on clearly Western European HTML probes — is it
+ * a few systematic bigrams or a diffuse accumulation of small
+ * contributions?</p>
+ */
+public final class InspectBigramContributions {
+
+ private InspectBigramContributions() {
+ }
+
+ public static void main(String[] args) throws Exception {
+ Path modelPath = null;
+ Path probeDir = null;
+ String classA = null;
+ String classB = null;
+ String[] probes = null;
+ int topK = 15;
+ boolean stripHtml = false;
+ for (int i = 0; i < args.length; i++) {
+ switch (args[i]) {
+ case "--model":
+ modelPath = Paths.get(args[++i]);
+ break;
+ case "--probe-dir":
+ probeDir = Paths.get(args[++i].replaceFirst("^~",
+ System.getProperty("user.home")));
+ break;
+ case "--class-a":
+ classA = args[++i];
+ break;
+ case "--class-b":
+ classB = args[++i];
+ break;
+ case "--probes":
+ probes = args[++i].split(",");
+ break;
+ case "--top-k":
+ topK = Integer.parseInt(args[++i]);
+ break;
+ case "--strip-html":
+ stripHtml = true;
+ break;
+ default:
+ System.err.println("Unknown arg: " + args[i]);
+ System.exit(1);
+ }
+ }
+ if (modelPath == null || probeDir == null || classA == null
+ || classB == null || probes == null) {
+ System.err.println("Usage: InspectBigramContributions --model <bin> "
+ + "--probe-dir <dir> --class-a <c> --class-b <c> "
+ + "--probes p1,p2,... [--top-k N]");
+ System.exit(1);
+ }
+ NaiveBayesBigramEncodingDetector det;
+ try (InputStream is = Files.newInputStream(modelPath)) {
+ det = new NaiveBayesBigramEncodingDetector(is);
+ }
+ String[] labels = det.getLabels();
+ Map<String, Integer> labelIdx = new HashMap<>();
+ for (int i = 0; i < labels.length; i++) labelIdx.put(labels[i], i);
+ Integer a = labelIdx.get(classA);
+ Integer b = labelIdx.get(classB);
+ if (a == null || b == null) {
+ System.err.println("Unknown class. Available: " + labelIdx.keySet());
+ System.exit(1);
+ }
+ Charset csA = safe(classA);
+ Charset csB = safe(classB);
+
+ for (String pid : probes) {
+ Path p = probeDir.resolve(pid);
+ if (!Files.exists(p)) {
+ System.err.println("Missing: " + p);
+ continue;
+ }
+ byte[] bytes = Files.readAllBytes(p);
+ int rawLen = bytes.length;
+ int stripTagCount = 0;
+ if (stripHtml) {
+ byte[] dst = new byte[bytes.length];
+ HtmlByteStripper.Result r = HtmlByteStripper.strip(bytes, 0, bytes.length, dst, 0);
+ stripTagCount = r.tagCount;
+ if (r.tagCount >= 1) {
+ byte[] trimmed = new byte[r.length];
+ System.arraycopy(dst, 0, trimmed, 0, r.length);
+ bytes = trimmed;
+ }
+ }
+ // Aggregate per-bigram across the probe.
+ List<NaiveBayesBigramEncodingDetector.BigramContrib> contribs =
+ det.analyzeBigrams(bytes, a, b);
+ // (bigram → [count, sumA, sumB])
+ TreeMap<Integer, double[]> agg = new TreeMap<>();
+ for (NaiveBayesBigramEncodingDetector.BigramContrib c : contribs) {
+ double[] e = agg.computeIfAbsent(c.bigram, k -> new double[3]);
+ e[0] += 1;
+ e[1] += c.contribA;
+ e[2] += c.contribB;
+ }
+ int totalScored = contribs.size();
+ int distinct = agg.size();
+ double sumA = 0;
+ double sumB = 0;
+ for (double[] e : agg.values()) {
+ sumA += e[1];
+ sumB += e[2];
+ }
+ double margin = (sumA - sumB) / Math.max(1, totalScored);
+ String short_ = pid.contains("/")
+ ? pid.substring(pid.indexOf('/') + 1, pid.indexOf('/') + 13) : pid;
+ System.out.printf(Locale.ROOT,
+ "=== %s raw=%d strip-tags=%d post-strip=%d scored=%d distinct=%d total[%s]=%+9.3f total[%s]=%+9.3f margin/bg=%+.4f ===%n",
+ short_, rawLen, stripTagCount, bytes.length, totalScored, distinct,
+ classA, sumA, classB, sumB, margin);
+
+ // Rank bigrams by signed accumulated diff (positive = pulls toward A).
+ java.util.List<Map.Entry<Integer, double[]>> entries =
+ new java.util.ArrayList<>(agg.entrySet());
+ entries.sort((x, y) -> Double.compare(
+ (y.getValue()[1] - y.getValue()[2]),
+ (x.getValue()[1] - x.getValue()[2])));
+
+ System.out.printf(Locale.ROOT,
+ " TOP-%d bigrams pulling toward %s (positive):%n", topK, classA);
+ printBlock(entries, 0, topK, csA, csB, classA, classB, true);
+ System.out.printf(Locale.ROOT,
+ " TOP-%d bigrams pulling toward %s (negative):%n", topK, classB);
+ printBlock(entries, entries.size() - topK, topK, csA, csB, classA, classB, false);
+ System.out.println();
+ }
+ }
+
+ private static void printBlock(List<Map.Entry<Integer, double[]>> entries,
+ int start, int n, Charset csA, Charset csB,
+ String classA, String classB, boolean fromHead) {
+ if (start < 0) start = 0;
+ int end = Math.min(start + n, entries.size());
+ if (fromHead) {
+ for (int i = start; i < end; i++) printRow(entries.get(i), csA, csB, classA, classB);
+ } else {
+ // print from end backwards (most-negative first)
+ for (int i = entries.size() - 1; i >= start; i--) {
+ printRow(entries.get(i), csA, csB, classA, classB);
+ }
+ }
+ }
+
+ private static void printRow(Map.Entry<Integer, double[]> e,
+ Charset csA, Charset csB,
+ String classA, String classB) {
+ int bg = e.getKey();
+ int b0 = (bg >>> 8) & 0xFF;
+ int b1 = bg & 0xFF;
+ double count = e.getValue()[0];
+ double a = e.getValue()[1];
+ double b = e.getValue()[2];
+ byte[] bytes = new byte[]{(byte) b0, (byte) b1};
+ String hi = ((b0 >= 0x80) ? "H" : "-") + ((b1 >= 0x80) ? "H" : "-");
+ System.out.printf(Locale.ROOT,
+ " %02X %02X %s n=%6.0f %s=%+9.3f %s=%+9.3f diff=%+8.3f decode(%s)='%s' decode(%s)='%s'%n",
+ b0, b1, hi, count, classA, a, classB, b, (a - b),
+ classA, decode(csA, bytes), classB, decode(csB, bytes));
+ }
+
+ private static Charset safe(String name) {
+ try { return Charset.forName(name); }
+ catch (Exception e) { return null; }
+ }
+
+ private static String decode(Charset cs, byte[] bytes) {
+ if (cs == null) return "?";
+ String s = new String(bytes, cs);
+ StringBuilder out = new StringBuilder();
+ for (int i = 0; i < s.length(); ) {
+ int cp = s.codePointAt(i);
+ if (cp < 0x20 || cp == 0x7F) out.append(String.format(Locale.ROOT, "\\x%02X", cp));
+ else if (cp == 0xFFFD) out.append("<?>");
+ else out.appendCodePoint(cp);
+ i += Character.charCount(cp);
+ }
+ return out.toString();
+ }
+}
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/TraceMojibuster.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/TraceMojibuster.java
new file mode 100644
index 0000000000..99017e7a3f
--- /dev/null
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/TraceMojibuster.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.chardetect;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.tika.detect.EncodingResult;
+
+/**
+ * Trace each layer of Mojibuster on a set of files: raw NB, post-strip NB,
+ * and full Mojibuster.detect(). Helps locate where a specific charset
+ * pick comes from in the pipeline.
+ */
+public final class TraceMojibuster {
+
+ private TraceMojibuster() {
+ }
+
+ public static void main(String[] args) throws Exception {
+ Path probeDir = null;
+ String[] probes = null;
+ Path inlineFile = null;
+ byte[] inlineBytes = null;
+ boolean fullRanking = false;
+ int rankingTopN = 25;
+ for (int i = 0; i < args.length; i++) {
+ switch (args[i]) {
+ case "--probe-dir":
+ probeDir = Paths.get(args[++i].replaceFirst("^~",
+ System.getProperty("user.home")));
+ break;
+ case "--probes":
+ probes = args[++i].split(",");
+ break;
+ case "--file":
+ inlineFile = Paths.get(args[++i].replaceFirst("^~",
+ System.getProperty("user.home")));
+ break;
+ case "--bytes-hex":
+ inlineBytes = decodeHex(args[++i]);
+ break;
+ case "--full-ranking":
+ fullRanking = true;
+ break;
+ case "--ranking-top-n":
+ rankingTopN = Integer.parseInt(args[++i]);
+ break;
+ default:
+ System.err.println("Unknown arg: " + args[i]);
+ System.exit(1);
+ }
+ }
+ boolean hasInline = inlineFile != null || inlineBytes != null;
+ boolean hasProbeList = probeDir != null && probes != null;
+ if (!hasInline && !hasProbeList) {
+ System.err.println("Usage: TraceMojibuster"
+ + " (--probe-dir <dir> --probes p1,p2,... | --file <path> | --bytes-hex <hex>)"
+ + " [--full-ranking] [--ranking-top-n N]");
+ System.exit(1);
+ }
+ // Load the bundled model from the classpath (same path Mojibuster uses).
+ NaiveBayesBigramEncodingDetector rawNb;
+ try (InputStream is = MojibusterEncodingDetector.class
+ .getResourceAsStream(
+ "/org/apache/tika/ml/chardetect/nb-bigram.bin")) {
+ if (is == null) throw new IOException("bundled model not on classpath");
+ rawNb = new NaiveBayesBigramEncodingDetector(is);
+ }
+ MojibusterEncodingDetector det = new MojibusterEncodingDetector();
+
+ // Build the (label, bytes) work list. Inline probes (--file /
+ // --bytes-hex) are processed first; then any --probes from
+ // --probe-dir.
+ List<ProbeInput> work = new ArrayList<>();
+ if (inlineFile != null) {
+ if (!Files.exists(inlineFile)) {
+ System.err.println("Missing: " + inlineFile);
+ System.exit(1);
+ }
+ work.add(new ProbeInput(inlineFile.getFileName().toString(),
+ Files.readAllBytes(inlineFile)));
+ }
+ if (inlineBytes != null) {
+ work.add(new ProbeInput("inline-hex(" + inlineBytes.length + "B)",
+ inlineBytes));
+ }
+ if (probes != null) {
+ for (String pid : probes) {
+ Path p = probeDir.resolve(pid);
+ if (!Files.exists(p)) {
+ System.err.println("Missing: " + p);
+ continue;
+ }
+ String shortId = pid.contains("/")
+ ? pid.substring(pid.indexOf('/') + 1, pid.indexOf('/') + 13) : pid;
+ work.add(new ProbeInput(shortId, Files.readAllBytes(p)));
+ }
+ }
+
+ for (ProbeInput w : work) {
+ byte[] bytes = w.bytes;
+ System.out.println();
+ System.out.println("==== " + w.label + " raw=" + bytes.length + " bytes ====");
+
+ // Layer 1: raw NB on raw bytes (no strip).
+ List<EncodingResult> rawResults = rawNb.detect(bytes);
+ System.out.println(" raw NB (no strip): " + fmt(rawResults));
+ if (fullRanking) {
+ dumpFullRanking(rawNb, bytes, "raw", rankingTopN);
+ }
+
+ // Layer 2: NB on HTML-stripped bytes.
+ byte[] dst = new byte[bytes.length];
+ HtmlByteStripper.Result sr = HtmlByteStripper.strip(bytes, 0, bytes.length, dst, 0);
+ byte[] strippedView = null;
+ if (sr.tagCount >= 1) {
+ strippedView = new byte[sr.length];
+ System.arraycopy(dst, 0, strippedView, 0, sr.length);
+ System.out.printf(Locale.ROOT,
+ " HTML strip: tags=%d, post-strip=%d bytes (%.1f%% kept)%n",
+ sr.tagCount, sr.length, 100.0 * sr.length / bytes.length);
+ List<EncodingResult> stripResults = rawNb.detect(strippedView);
+ System.out.println(" NB on stripped bytes: " + fmt(stripResults));
+ if (fullRanking) {
+ dumpFullRanking(rawNb, strippedView, "strip", rankingTopN);
+ }
+ } else {
+ System.out.println(" HTML strip: tagCount=0 (backoff, used original)");
+ }
+
+ // Layer 3: full Mojibuster (which internally strips conditionally).
+ List<EncodingResult> mojiResults = det.detect(bytes);
+ System.out.println(" Full Mojibuster.detect: " + fmt(mojiResults));
+ }
+ }
+
+ /**
+ * Print every class sorted by raw NB log-score for this probe.
+ * Shows where the true charset actually ranks before margin gating,
+ * gap-from-top-1 in nats, and gap-per-scored-bigram (the unit the
+ * margin gate uses). An "emit?" column flags which candidates would
+ * pass the {@link NaiveBayesBigramEncodingDetector#MARGIN_THRESHOLD_NATS_PER_BIGRAM}
+ * gate.
+ */
+ private static void dumpFullRanking(NaiveBayesBigramEncodingDetector nb,
+ byte[] probe, String layer, int topN) {
+ NaiveBayesBigramEncodingDetector.ScoreResult sr = nb.scoreClassesAndCount(probe);
+ if (sr == null) {
+ System.out.println(" [full-ranking " + layer + "] <probe too short to score>");
+ return;
+ }
+ String[] labels = nb.getLabels();
+ int n = labels.length;
+ Integer[] idx = new Integer[n];
+ for (int i = 0; i < n; i++) idx[i] = i;
+ final double[] scores = sr.scores;
+ java.util.Arrays.sort(idx, (a, b) -> Double.compare(scores[b], scores[a]));
+ double top1 = scores[idx[0]];
+ double marginNats = NaiveBayesBigramEncodingDetector.MARGIN_THRESHOLD_NATS_PER_BIGRAM
+ * Math.max(1, sr.scoredBigrams);
+ System.out.printf(Locale.ROOT,
+ " [full-ranking %s] scoredBigrams=%d totalBigrams=%d marginGate=%.3f nats (%.3f×bg)%n",
+ layer, sr.scoredBigrams, sr.totalBigrams, marginNats,
+ NaiveBayesBigramEncodingDetector.MARGIN_THRESHOLD_NATS_PER_BIGRAM);
+ int limit = Math.min(topN, n);
+ for (int rank = 0; rank < limit; rank++) {
+ int c = idx[rank];
+ double score = scores[c];
+ double gap = top1 - score;
+ double gapPerBg = (sr.scoredBigrams > 0) ? gap / sr.scoredBigrams : Double.NaN;
+ String emit = (rank == 0) ? "top1"
+ : (gap < marginNats ? "EMIT" : "----");
+ System.out.printf(Locale.ROOT,
+ " #%2d %-18s score=%+11.3f gap=%+8.3f gap/bg=%+.4f %s%n",
+ rank + 1, labels[c], score, gap, gapPerBg, emit);
+ }
+ }
+
+ private static byte[] decodeHex(String s) {
+ String cleaned = s.replaceAll("[\\s,:]", "");
+ if (cleaned.length() % 2 != 0) {
+ throw new IllegalArgumentException("hex string must have even length: " + s);
+ }
+ byte[] out = new byte[cleaned.length() / 2];
+ for (int i = 0; i < out.length; i++) {
+ out[i] = (byte) Integer.parseInt(cleaned.substring(2 * i, 2 * i + 2), 16);
+ }
+ return out;
+ }
+
+ private static final class ProbeInput {
+ final String label;
+ final byte[] bytes;
+ ProbeInput(String label, byte[] bytes) {
+ this.label = label;
+ this.bytes = bytes;
+ }
+ }
+
+ private static String fmt(List<EncodingResult> rs) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < rs.size(); i++) {
+ if (i > 0) sb.append(", ");
+ EncodingResult r = rs.get(i);
+ sb.append(r.getCharset().name())
+ .append("@").append(String.format(Locale.ROOT, "%.3f", r.getConfidence()))
+ .append("/").append(r.getResultType());
+ }
+ if (sb.length() == 0) sb.append("<empty>");
+ return sb.toString();
+ }
+}
diff --git a/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/tools/BuildCharsetTrainingData.java b/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/tools/BuildCharsetTrainingData.java
index bf3efa9fe6..f863456a69 100644
--- a/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/tools/BuildCharsetTrainingData.java
+++ b/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/tools/BuildCharsetTrainingData.java
@@ -214,16 +214,21 @@ public class BuildCharsetTrainingData {
put("mlt", "ISO-8859-3");
put("tur", "windows-1254");
// Central / Eastern European
- put("ces", "windows-1250", "IBM852");
- put("pol", "windows-1250", "IBM852");
- put("hrv", "windows-1250", "IBM852");
- put("slk", "windows-1250", "IBM852");
- put("slv", "windows-1250", "IBM852");
- put("hun", "windows-1250", "IBM852");
+ // ISO-8859-2 (Latin-2) added 2026-05-18 alongside the
+ // windows-1250 superset for Polish / Czech / Slovak / Slovenian /
+ // Croatian / Hungarian — gives the model a separate class so it
+ // can discriminate genuinely-ISO-8859-2 content (e.g. files with
+ // explicit meta declarations) from windows-1250 supersets.
+ put("ces", "windows-1250", "IBM852", "ISO-8859-2");
+ put("pol", "windows-1250", "IBM852", "ISO-8859-2");
+ put("hrv", "windows-1250", "IBM852", "ISO-8859-2");
+ put("slk", "windows-1250", "IBM852", "ISO-8859-2");
+ put("slv", "windows-1250", "IBM852", "ISO-8859-2");
+ put("hun", "windows-1250", "IBM852", "ISO-8859-2");
// ISO-8859-16 (Latin-10) retained for Romanian and Albanian
- put("ron", "windows-1250", "IBM852", "ISO-8859-16");
- put("bos", "windows-1250", "IBM852");
- put("sqi", "windows-1250", "IBM852", "ISO-8859-16");
+ put("ron", "windows-1250", "IBM852", "ISO-8859-2", "ISO-8859-16");
+ put("bos", "windows-1250", "IBM852", "ISO-8859-2");
+ put("sqi", "windows-1250", "IBM852", "ISO-8859-2", "ISO-8859-16");
// Cyrillic — keep all distinct encodings
put("rus", "windows-1251", "KOI8-R", "IBM855", "IBM866", "x-mac-cyrillic");
put("ukr", "windows-1251", "KOI8-U", "IBM855", "x-mac-cyrillic");
diff --git a/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/tools/DiagnoseDiscrimination.java b/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/tools/DiagnoseDiscrimination.java
new file mode 100644
index 0000000000..29a16e517e
--- /dev/null
+++ b/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/tools/DiagnoseDiscrimination.java
@@ -0,0 +1,399 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.chardetect.tools;
+
+import java.io.DataInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.zip.GZIPInputStream;
+
+/**
+ * Diagnostic for the win-1252-vs-MacRoman / IBM850 / windows-1250 /
+ * windows-1257 / etc. confusion problem.
+ *
+ * <p>Reads {@code <charset>.bin.gz} files from a training directory,
+ * counts per-byte and per-bigram frequencies for the requested classes,
+ * then for each pair (A, B) prints the top-K bytes and bigrams by
+ * Kullback-Leibler-divergence contribution:</p>
+ *
+ * <pre>
+ * KL_contrib(b) = (p_A(b) - p_B(b)) * log(p_A(b) / p_B(b))
+ * </pre>
+ *
+ * <p>The bytes / bigrams with the largest contributions are the ones
+ * whose frequencies differ most between A and B — i.e., the bytes /
+ * bigrams the NB classifier should rely on to tell A from B.</p>
+ *
+ * <p>Annotates each output line with:</p>
+ * <ul>
+ * <li>the byte's hex value</li>
+ * <li>what Unicode character the byte (or bigram) decodes to under
+ * each charset — to see whether the discriminative bytes are
+ * letters, punctuation, smart quotes, controls, etc.</li>
+ * <li>per-class probability of seeing this byte</li>
+ * </ul>
+ */
+public final class DiagnoseDiscrimination {
+
+ private static final int BIGRAM_SPACE = 65536;
+
+ private DiagnoseDiscrimination() {
+ }
+
+ public static void main(String[] args) throws IOException {
+ Path dataDir = null;
+ List<String> classes = new ArrayList<>();
+ int topK = 20;
+ int maxSamplesPerClass = 50_000;
+ for (int i = 0; i < args.length; i++) {
+ switch (args[i]) {
+ case "--data":
+ dataDir = Paths.get(args[++i]);
+ break;
+ case "--classes":
+ classes = Arrays.asList(args[++i].split(","));
+ break;
+ case "--top-k":
+ topK = Integer.parseInt(args[++i]);
+ break;
+ case "--max-samples-per-class":
+ maxSamplesPerClass = Integer.parseInt(args[++i]);
+ break;
+ default:
+ System.err.println("Unknown arg: " + args[i]);
+ System.exit(1);
+ }
+ }
+ if (dataDir == null || classes.isEmpty()) {
+ System.err.println("Usage: DiagnoseDiscrimination --data <dir>"
+ + " --classes c1,c2,c3 [--top-k 20]"
+ + " [--max-samples-per-class N]");
+ System.exit(1);
+ }
+
+ Map<String, long[]> byteCounts = new LinkedHashMap<>();
+ Map<String, long[]> bigramCounts = new LinkedHashMap<>();
+ Map<String, Long> byteTotals = new LinkedHashMap<>();
+ Map<String, Long> bigramTotals = new LinkedHashMap<>();
+
+ for (String label : classes) {
+ Path f = dataDir.resolve(label + ".bin.gz");
+ if (!Files.exists(f)) {
+ System.err.println("Missing: " + f);
+ continue;
+ }
+ long[] bc = new long[256];
+ long[] bgc = new long[BIGRAM_SPACE];
+ long bt = 0, bgt = 0;
+ int sampled = 0;
+ try (InputStream fis = new FileInputStream(f.toFile());
+ GZIPInputStream gis = new GZIPInputStream(fis);
+ DataInputStream dis = new DataInputStream(gis)) {
+ while (sampled < maxSamplesPerClass) {
+ int len;
+ try {
+ len = dis.readUnsignedShort();
+ } catch (java.io.EOFException eof) {
+ break;
+ }
+ byte[] s = new byte[len];
+ dis.readFully(s);
+ for (int i = 0; i < s.length; i++) {
+ bc[s[i] & 0xFF]++;
+ bt++;
+ }
+ for (int i = 0; i + 1 < s.length; i++) {
+ int bg = ((s[i] & 0xFF) << 8) | (s[i + 1] & 0xFF);
+ bgc[bg]++;
+ bgt++;
+ }
+ sampled++;
+ }
+ }
+ byteCounts.put(label, bc);
+ bigramCounts.put(label, bgc);
+ byteTotals.put(label, bt);
+ bigramTotals.put(label, bgt);
+ System.out.printf(Locale.ROOT, "loaded %-15s %,d samples %,d bytes %,d bigrams%n",
+ label, sampled, bt, bgt);
+ }
+
+ System.out.println();
+ for (int i = 0; i < classes.size(); i++) {
+ for (int j = i + 1; j < classes.size(); j++) {
+ String a = classes.get(i);
+ String b = classes.get(j);
+ if (!byteCounts.containsKey(a) || !byteCounts.containsKey(b)) {
+ continue;
+ }
+ System.out.println("=========================================");
+ System.out.printf(Locale.ROOT, " %s vs %s%n", a, b);
+ System.out.println("=========================================");
+ System.out.println();
+ printTopBytes(a, b,
+ byteCounts.get(a), byteCounts.get(b),
+ byteTotals.get(a), byteTotals.get(b),
+ topK);
+ System.out.println();
+ printTopBigrams(a, b,
+ bigramCounts.get(a), bigramCounts.get(b),
+ bigramTotals.get(a), bigramTotals.get(b),
+ topK);
+ System.out.println();
+ printExpectedContribution(a, b,
+ bigramCounts.get(a), bigramCounts.get(b),
+ bigramTotals.get(a), bigramTotals.get(b),
+ topK);
+ System.out.println();
+ }
+ }
+ }
+
+ private static void printTopBytes(String a, String b,
+ long[] ca, long[] cb,
+ long ta, long tb, int topK) {
+ Charset csA = safeCharset(a);
+ Charset csB = safeCharset(b);
+ double[] kl = new double[256];
+ for (int x = 0; x < 256; x++) {
+ double pa = (ca[x] + 0.5) / (ta + 128.0);
+ double pb = (cb[x] + 0.5) / (tb + 128.0);
+ kl[x] = (pa - pb) * Math.log(pa / pb);
+ }
+ Integer[] idx = new Integer[256];
+ for (int x = 0; x < 256; x++) idx[x] = x;
+ Arrays.sort(idx, (x, y) -> Double.compare(kl[y], kl[x]));
+
+ System.out.printf(Locale.ROOT,
+ " Top-%d discriminative single bytes (by KL contribution):%n", topK);
+ System.out.printf(Locale.ROOT,
+ " %-6s %-6s %-12s %-12s %-15s %-15s %-8s%n",
+ "byte", "high?", "p(" + a + ")", "p(" + b + ")", "decode(" + a + ")",
+ "decode(" + b + ")", "KL");
+ for (int k = 0; k < topK; k++) {
+ int x = idx[k];
+ double pa = (ca[x] + 0.5) / (ta + 128.0);
+ double pb = (cb[x] + 0.5) / (tb + 128.0);
+ String dA = decode(csA, new byte[]{(byte) x});
+ String dB = decode(csB, new byte[]{(byte) x});
+ String alpha = "";
+ if (x >= 0x80) {
+ boolean alphaA = !dA.isEmpty() && Character.isLetter(dA.codePointAt(0));
+ boolean alphaB = !dB.isEmpty() && Character.isLetter(dB.codePointAt(0));
+ alpha = (alphaA ? "A" : "-") + (alphaB ? "B" : "-");
+ }
+ System.out.printf(Locale.ROOT,
+ " 0x%02X %-6s %-12.6f %-12.6f %-15s %-15s %-8.4f %s%n",
+ x, (x >= 0x80) ? "HI" : "",
+ pa, pb, prettify(dA), prettify(dB), kl[x], alpha);
+ }
+ }
+
+ private static void printTopBigrams(String a, String b,
+ long[] ca, long[] cb,
+ long ta, long tb, int topK) {
+ Charset csA = safeCharset(a);
+ Charset csB = safeCharset(b);
+ double[] kl = new double[BIGRAM_SPACE];
+ for (int x = 0; x < BIGRAM_SPACE; x++) {
+ if (ca[x] == 0 && cb[x] == 0) continue;
+ double pa = (ca[x] + 0.5) / (ta + BIGRAM_SPACE * 0.5);
+ double pb = (cb[x] + 0.5) / (tb + BIGRAM_SPACE * 0.5);
+ kl[x] = (pa - pb) * Math.log(pa / pb);
+ }
+ Integer[] idx = new Integer[BIGRAM_SPACE];
+ for (int x = 0; x < BIGRAM_SPACE; x++) idx[x] = x;
+ Arrays.sort(idx, (x, y) -> Double.compare(kl[y], kl[x]));
+
+ System.out.printf(Locale.ROOT,
+ " Top-%d discriminative bigrams (by KL contribution):%n", topK);
+ System.out.printf(Locale.ROOT,
+ " %-9s %-9s %-12s %-12s %-15s %-15s%n",
+ "bigram", "high?", "p(" + a + ")", "p(" + b + ")",
+ "decode(" + a + ")", "decode(" + b + ")");
+ for (int k = 0; k < topK; k++) {
+ int bg = idx[k];
+ int b0 = (bg >>> 8) & 0xFF;
+ int b1 = bg & 0xFF;
+ double pa = (ca[bg] + 0.5) / (ta + BIGRAM_SPACE * 0.5);
+ double pb = (cb[bg] + 0.5) / (tb + BIGRAM_SPACE * 0.5);
+ byte[] bytes = new byte[]{(byte) b0, (byte) b1};
+ String hi = ((b0 >= 0x80) ? "H" : "-") + ((b1 >= 0x80) ? "H" : "-");
+ System.out.printf(Locale.ROOT,
+ " %02X %02X %-9s %-12.6f %-12.6f %-15s %-15s KL=%.4f%n",
+ b0, b1, hi, pa, pb,
+ prettify(decode(csA, bytes)),
+ prettify(decode(csB, bytes)),
+ kl[bg]);
+ }
+ }
+
+ /**
+ * For each bigram, compute the SIGNED expected contribution to a
+ * probe-from-A's score-margin vs B:
+ *
+ * contrib(bg) = p_A(bg) * (log p_A(bg) − log p_B(bg))
+ *
+ * Then aggregate over byte-pair categories (ASCII-only, single-high,
+ * double-high, whitespace-anchored, control-anchored) so we can see
+ * whether the model's score on a typical A-probe is being driven by
+ * "real" high-byte discriminators or by dumb high-frequency ASCII
+ * bigrams whose per-class probability happens to be slightly
+ * skewed. Also prints the top-K SIGNED contributors in each
+ * direction.
+ *
+ * <p>Negative entries are "if probe is class A, this bigram pulls
+ * the score toward B" — exactly the bigrams that would silently
+ * bias the decision the wrong way.</p>
+ */
+ private static void printExpectedContribution(String a, String b,
+ long[] ca, long[] cb,
+ long ta, long tb, int topK) {
+ Charset csA = safeCharset(a);
+ Charset csB = safeCharset(b);
+ double[] contrib = new double[BIGRAM_SPACE];
+ double asciiSum = 0, singleHiSum = 0, doubleHiSum = 0;
+ double wsAnchoredSum = 0, ctrlAnchoredSum = 0;
+ double asciiAbs = 0, singleHiAbs = 0, doubleHiAbs = 0;
+ for (int bg = 0; bg < BIGRAM_SPACE; bg++) {
+ int b0 = (bg >>> 8) & 0xFF;
+ int b1 = bg & 0xFF;
+ double pa = (ca[bg] + 0.5) / (ta + BIGRAM_SPACE * 0.5);
+ double pb = (cb[bg] + 0.5) / (tb + BIGRAM_SPACE * 0.5);
+ double c = pa * (Math.log(pa) - Math.log(pb));
+ contrib[bg] = c;
+ int hi = ((b0 >= 0x80) ? 1 : 0) + ((b1 >= 0x80) ? 1 : 0);
+ boolean ws0 = isWhitespaceByte(b0);
+ boolean ws1 = isWhitespaceByte(b1);
+ boolean ctrl0 = b0 < 0x20 && !ws0;
+ boolean ctrl1 = b1 < 0x20 && !ws1;
+ if (hi == 0) {
+ asciiSum += c;
+ asciiAbs += Math.abs(c);
+ } else if (hi == 1) {
+ singleHiSum += c;
+ singleHiAbs += Math.abs(c);
+ } else {
+ doubleHiSum += c;
+ doubleHiAbs += Math.abs(c);
+ }
+ if (ws0 || ws1) wsAnchoredSum += c;
+ if (ctrl0 || ctrl1) ctrlAnchoredSum += c;
+ }
+
+ System.out.printf(Locale.ROOT,
+ " Expected per-probe-bigram score contribution (probe-from-%s, vs %s):%n", a, b);
+ System.out.printf(Locale.ROOT,
+ " Positive = pushes toward %s (correct). Negative = pushes toward %s.%n%n", a, b);
+ System.out.printf(Locale.ROOT,
+ " Category sum abs-sum net-direction%n");
+ printCat("ASCII pairs (both < 0x80)", asciiSum, asciiAbs, a, b);
+ printCat("Single-high-byte bigrams ", singleHiSum, singleHiAbs, a, b);
+ printCat("Double-high-byte bigrams ", doubleHiSum, doubleHiAbs, a, b);
+ printCat("Whitespace-anchored ", wsAnchoredSum, 0.0, a, b);
+ printCat("Control-anchored (<0x20) ", ctrlAnchoredSum, 0.0, a, b);
+ double total = asciiSum + singleHiSum + doubleHiSum;
+ System.out.printf(Locale.ROOT,
+ " TOTAL (KL(%s||%s)): %+.6f%n", a, b, total);
+
+ // Top-K NEGATIVE contributors: bigrams pulling toward B on A-probes.
+ Integer[] idx = new Integer[BIGRAM_SPACE];
+ for (int x = 0; x < BIGRAM_SPACE; x++) idx[x] = x;
+ Arrays.sort(idx, (x, y) -> Double.compare(contrib[x], contrib[y]));
+ System.out.printf(Locale.ROOT,
+ "%n Top-%d bigrams PULLING TOWARD %s on a typical %s probe:%n", topK, b, a);
+ System.out.printf(Locale.ROOT,
+ " %-9s %-9s %-15s %-15s %-12s%n",
+ "bigram", "hi?", "decode(" + a + ")", "decode(" + b + ")", "contrib");
+ for (int k = 0; k < topK; k++) {
+ int bg = idx[k];
+ if (contrib[bg] >= 0) break;
+ int b0 = (bg >>> 8) & 0xFF;
+ int b1 = bg & 0xFF;
+ String hi = ((b0 >= 0x80) ? "H" : "-") + ((b1 >= 0x80) ? "H" : "-");
+ System.out.printf(Locale.ROOT,
+ " %02X %02X %-9s %-15s %-15s %+.6f%n",
+ b0, b1, hi,
+ prettify(decode(csA, new byte[]{(byte)b0, (byte)b1})),
+ prettify(decode(csB, new byte[]{(byte)b0, (byte)b1})),
+ contrib[bg]);
+ }
+ }
+
+ private static void printCat(String label, double signedSum,
+ double absSum, String a, String b) {
+ if (absSum > 0) {
+ System.out.printf(Locale.ROOT,
+ " %-30s %+.6f %.6f %s%n",
+ label, signedSum, absSum, signedSum > 0 ? "→ " + a : "→ " + b);
+ } else {
+ System.out.printf(Locale.ROOT,
+ " %-30s %+.6f %s%n",
+ label, signedSum, signedSum > 0 ? "→ " + a : "→ " + b);
+ }
+ }
+
+ private static boolean isWhitespaceByte(int b) {
+ return b == 0x09 || b == 0x0A || b == 0x0B || b == 0x0C
+ || b == 0x0D || b == 0x20;
+ }
+
+ private static Charset safeCharset(String name) {
+ try {
+ return Charset.forName(name);
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ private static String decode(Charset cs, byte[] bytes) {
+ if (cs == null) return "?";
+ try {
+ return new String(bytes, cs);
+ } catch (Exception e) {
+ return "?";
+ }
+ }
+
+ private static String prettify(String s) {
+ if (s == null || s.isEmpty()) return "<empty>";
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < s.length(); ) {
+ int cp = s.codePointAt(i);
+ if (cp < 0x20 || cp == 0x7F) {
+ sb.append(String.format("\\x%02X", cp));
+ } else if (cp == 0xFFFD) {
+ sb.append("<FFFD>");
+ } else {
+ sb.appendCodePoint(cp);
+ }
+ i += Character.charCount(cp);
+ }
+ return sb.toString();
+ }
+}
diff --git a/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/tools/RebalanceCharsetTraining.java b/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/tools/RebalanceCharsetTraining.java
new file mode 100644
index 0000000000..c81541840c
--- /dev/null
+++ b/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/tools/RebalanceCharsetTraining.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.chardetect.tools;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CodingErrorAction;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.LinkedHashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+/**
+ * Rebalance the per-class training corpus so all Western European Latin
+ * SBCS classes have the same underlying source text — eliminating the
+ * "common ASCII bigrams pull toward MacRoman" bias surfaced by
+ * {@link DiagnoseDiscrimination}.
+ *
+ * <p>Reads {@code windows-1252.bin.gz} (which contains samples drawn
+ * from all 19 Western European languages), decodes each sample under
+ * windows-1252 to get the original text, then re-encodes that text
+ * under each target charset (MacRoman / IBM850 / IBM500 / IBM1047),
+ * writing new {@code <target>.bin.gz} files. Samples whose codepoints
+ * cannot be encoded under a target are skipped (handled via the encoder's
+ * REPORT action — keeps the per-class distribution clean rather than
+ * importing replacement chars that aren't part of the target's natural
+ * vocabulary).</p>
+ *
+ * <p>Result: MacRoman / IBM850 / IBM500 / IBM1047 see the SAME source
+ * text as windows-1252, just encoded differently — so the only
+ * cross-class differences are byte-position differences, which is
+ * exactly what the model should be learning.</p>
+ *
+ * <p>Usage:</p>
+ * <pre>
+ * java RebalanceCharsetTraining \
+ * --train-dir <dir containing windows-1252.bin.gz> \
+ * --output-dir <dir to write re-encoded .bin.gz files> \
+ * [--targets x-MacRoman,IBM850,IBM500,IBM1047]
+ * </pre>
+ */
+public final class RebalanceCharsetTraining {
+
+ private static final int MAX_SAMPLE_BYTES = 65_535;
+
+ private RebalanceCharsetTraining() {
+ }
+
+ public static void main(String[] args) throws IOException {
+ Path trainDir = null;
+ Path outputDir = null;
+ String[] targets = {"x-MacRoman", "IBM850", "IBM500", "IBM1047"};
+ for (int i = 0; i < args.length; i++) {
+ switch (args[i]) {
+ case "--train-dir":
+ trainDir = Paths.get(args[++i]);
+ break;
+ case "--output-dir":
+ outputDir = Paths.get(args[++i]);
+ break;
+ case "--targets":
+ targets = args[++i].split(",");
+ break;
+ default:
+ System.err.println("Unknown arg: " + args[i]);
+ System.exit(1);
+ }
+ }
+ if (trainDir == null || outputDir == null) {
+ System.err.println("Usage: RebalanceCharsetTraining"
+ + " --train-dir <dir> --output-dir <dir>"
+ + " [--targets x-MacRoman,IBM850,IBM500,IBM1047]");
+ System.exit(1);
+ }
+ Files.createDirectories(outputDir);
+
+ Path source = trainDir.resolve("windows-1252.bin.gz");
+ if (!Files.exists(source)) {
+ System.err.println("No source: " + source);
+ System.exit(1);
+ }
+ Charset win1252 = Charset.forName("windows-1252");
+
+ Map<String, CharsetEncoder> encoders = new LinkedHashMap<>();
+ Map<String, DataOutputStream> outs = new LinkedHashMap<>();
+ Map<String, long[]> stats = new LinkedHashMap<>();
+ for (String t : targets) {
+ String javaName = toJavaCharsetName(t);
+ Charset cs;
+ try {
+ cs = Charset.forName(javaName);
+ } catch (Exception ex) {
+ System.err.println("Unsupported charset: " + t + " (java=" + javaName + ")");
+ continue;
+ }
+ // IGNORE drops unencodable codepoints (e.g. windows-1252's
+ // Š / ž / Czech letters that MacRoman / IBM850 don't have)
+ // but keeps the rest of the sample. Preserves bigram
+ // statistics for the encodable substring instead of throwing
+ // away the whole sample.
+ CharsetEncoder enc = cs.newEncoder()
+ .onUnmappableCharacter(CodingErrorAction.IGNORE)
+ .onMalformedInput(CodingErrorAction.IGNORE);
+ encoders.put(t, enc);
+ Path outFile = outputDir.resolve(t + ".bin.gz");
+ DataOutputStream dos = new DataOutputStream(
+ new GZIPOutputStream(new FileOutputStream(outFile.toFile())));
+ outs.put(t, dos);
+ stats.put(t, new long[]{0, 0, 0}); // kept, dropped, bytesWritten
+ }
+
+ long sourceSamples = 0;
+ try (DataInputStream dis = new DataInputStream(
+ new GZIPInputStream(new FileInputStream(source.toFile())))) {
+ while (true) {
+ int len;
+ try {
+ len = dis.readUnsignedShort();
+ } catch (java.io.EOFException eof) {
+ break;
+ }
+ byte[] sample = new byte[len];
+ dis.readFully(sample);
+ String text = new String(sample, win1252);
+ sourceSamples++;
+ for (Map.Entry<String, CharsetEncoder> e : encoders.entrySet()) {
+ String label = e.getKey();
+ CharsetEncoder enc = e.getValue();
+ enc.reset();
+ long[] s = stats.get(label);
+ try {
+ ByteBuffer bb = enc.encode(CharBuffer.wrap(text));
+ byte[] re = new byte[bb.remaining()];
+ bb.get(re);
+ if (re.length == 0 || re.length > MAX_SAMPLE_BYTES) {
+ s[1]++;
+ continue;
+ }
+ DataOutputStream dos = outs.get(label);
+ dos.writeShort(re.length);
+ dos.write(re);
+ s[0]++;
+ s[2] += re.length;
+ } catch (Exception ex) {
+ s[1]++;
+ }
+ }
+ }
+ }
+
+ for (DataOutputStream d : outs.values()) {
+ d.close();
+ }
+ System.out.printf(Locale.ROOT, "%nRead %,d source samples from %s%n",
+ sourceSamples, source.getFileName());
+ for (String t : targets) {
+ long[] s = stats.get(t);
+ if (s == null) continue;
+ System.out.printf(Locale.ROOT,
+ " %-15s kept=%,d dropped=%,d bytes=%,d → %s%n",
+ t, s[0], s[1], s[2], outputDir.resolve(t + ".bin.gz"));
+ }
+ System.out.println();
+ System.out.println("Other classes in " + trainDir + " are unchanged.");
+ System.out.println("To train on the rebalanced data, point TrainNaiveBayesBigram at"
+ + " a dir that combines this output with the other unchanged classes.");
+ }
+
+ /**
+ * Mirrors the training-label → Java charset mapping in
+ * BuildCharsetTrainingData / NaiveBayesBigramEncodingDetector.
+ */
+ private static String toJavaCharsetName(String label) {
+ switch (label) {
+ case "x-mac-cyrillic": return "x-MacCyrillic";
+ case "windows-874": return "x-windows-874";
+ case "IBM420-ltr":
+ case "IBM420-rtl": return "IBM420";
+ case "IBM424-ltr":
+ case "IBM424-rtl": return "IBM424";
+ default: return label;
+ }
+ }
+}
diff --git a/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/tools/TrainNaiveBayesBigram.java b/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/tools/TrainNaiveBayesBigram.java
index 12ac846a84..a082f4c5be 100644
--- a/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/tools/TrainNaiveBayesBigram.java
+++ b/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/tools/TrainNaiveBayesBigram.java
@@ -37,6 +37,8 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.zip.GZIPInputStream;
+import org.apache.tika.ml.chardetect.HtmlByteStripper;
+
/**
* Naive-Bayes byte-bigram charset classifier trainer.
*
@@ -80,6 +82,10 @@ public class TrainNaiveBayesBigram {
private static final double DEFAULT_ALPHA_BASE = 1.0;
private static final int DEFAULT_MAX_SAMPLES = 50_000;
+ /** Match {@code MojibusterEncodingDetector.MIN_TAG_COUNT_TO_USE_STRIP}. */
+ private static final int MIN_TAG_COUNT_TO_USE_STRIP = 1;
+ /** Match {@code MojibusterEncodingDetector.MIN_ENTITY_COUNT_TO_USE_STRIP}. */
+ private static final int MIN_ENTITY_COUNT_TO_USE_STRIP = 3;
/**
* Default per-class vocabulary coverage. 0.999 means the top-K
* most frequent bigrams covering 99.9% of the class's marginal
@@ -104,8 +110,7 @@ public class TrainNaiveBayesBigram {
* UTF-16-encoded CJK from legacy byte encodings because
* CJK characters in UTF-16 encode to byte pairs that alias
* common ASCII bigrams (e.g. U+6572 in UTF-16-LE is
- * {@code 72 65} which also encodes "re"). See
- * {@code ~/Desktop/claude-todo/charset/why-stride1-bigrams-dont-work-for-utf16.md}.</li>
+ * {@code 72 65} which also encodes "re").</li>
* <li>UTF-8 — trained NB class. UTF-8's lead + continuation
* byte-pair structure has distinctive frequency signatures
* that don't alias with legacy encodings at the bigram-
@@ -132,16 +137,20 @@ public class TrainNaiveBayesBigram {
"windows-1250", "windows-1251", "windows-1252", "windows-1253",
"windows-1254", "windows-1255", "windows-1256", "windows-1257",
"windows-1258", "windows-874",
- // ISO-8859 (2)
- "ISO-8859-3", "ISO-8859-16",
+ // ISO-8859 (3) — ISO-8859-2 added 2026-05-18 for Polish /
+ // Central European coverage alongside the windows-1250 superset
+ "ISO-8859-2", "ISO-8859-3", "ISO-8859-16",
// Mac (2)
"x-MacRoman", "x-mac-cyrillic");
/**
* Training-data filename → training-class-label aliases. Empty
* by default; reserved for cases where multiple training files
- * should merge into one class label. (UTF-16 was an experiment
- * that didn't pan out — see why-stride1-bigrams-dont-work-for-utf16.md.)
+ * should merge into one class label. (UTF-16-LE / UTF-16-BE were
+ * previously aliased to a single UTF-16 class but the experiment
+ * didn't pan out: stride-1 byte bigrams cannot discriminate UTF-16
+ * from legacy encodings — CJK characters in UTF-16 produce byte
+ * pairs that alias common ASCII bigrams.)
*/
static final Map<String, String> TRAINING_LABEL_ALIASES = Map.of();
@@ -189,7 +198,7 @@ public class TrainNaiveBayesBigram {
}
System.out.printf(Locale.ROOT,
- "coverage=%.3f alpha-base=%.3f max-samples/class=%,d%n",
+ "coverage=%.3f alpha-base=%.3f max-samples/class=%,d (markup-stripped via HtmlByteStripper)%n",
coverage, alphaBase, maxSamples);
System.out.println("Classes (" + classFilter.size() + "): "
+ new java.util.TreeSet<>(classFilter));
@@ -241,8 +250,10 @@ public class TrainNaiveBayesBigram {
// across contributing files so a 2-file class (UTF-16 LE+BE)
// doesn't overrun a 1-file class's sample count.
int perFileBudget = Math.max(1, maxSamples / entry.getValue().size());
+ long samplesStripped = 0;
for (Path f : entry.getValue()) {
int fileSamples = 0;
+ byte[] stripBuf = new byte[65536];
try (InputStream fis = new FileInputStream(f.toFile());
GZIPInputStream gis = new GZIPInputStream(fis);
DataInputStream dis = new DataInputStream(gis)) {
@@ -255,8 +266,30 @@ public class TrainNaiveBayesBigram {
}
byte[] sample = new byte[len];
dis.readFully(sample);
- for (int i = 0; i + 1 < sample.length; i++) {
- int bigram = ((sample[i] & 0xFF) << 8) | (sample[i + 1] & 0xFF);
+ // Train/inference symmetry: same HtmlByteStripper +
+ // same gate as MojibusterEncodingDetector at
+ // detect-time. Hardcoded ON to prevent accidental
+ // misconfiguration; the bigram tables MUST reflect
+ // the byte distribution NB will actually score.
+ if (stripBuf.length < len) {
+ stripBuf = new byte[len];
+ }
+ HtmlByteStripper.Result r = HtmlByteStripper.strip(
+ sample, 0, len, stripBuf, 0);
+ byte[] scoreBytes;
+ int scoreLen;
+ if (r.tagCount >= MIN_TAG_COUNT_TO_USE_STRIP
+ || r.entityCount >= MIN_ENTITY_COUNT_TO_USE_STRIP) {
+ scoreBytes = stripBuf;
+ scoreLen = r.length;
+ samplesStripped++;
+ } else {
+ scoreBytes = sample;
+ scoreLen = len;
+ }
+ for (int i = 0; i + 1 < scoreLen; i++) {
+ int bigram = ((scoreBytes[i] & 0xFF) << 8)
+ | (scoreBytes[i + 1] & 0xFF);
counts[bigram]++;
totalBigrams++;
}
@@ -268,10 +301,11 @@ public class TrainNaiveBayesBigram {
countsPerClass[ci] = counts;
totalsPerClass[ci] = totalBigrams;
System.out.printf(Locale.ROOT,
- " counted %-20s %,7d samples %,10d total bigrams (%d file%s)%n",
+ " counted %-20s %,7d samples %,10d total bigrams (%d file%s) stripped=%,d%n",
labels[ci], numSamples, totalBigrams,
entry.getValue().size(),
- entry.getValue().size() == 1 ? "" : "s");
+ entry.getValue().size() == 1 ? "" : "s",
+ samplesStripped);
ci++;
}
diff --git a/tika-ml/tika-ml-junkdetect/pom.xml b/tika-ml/tika-ml-junkdetect/pom.xml
index a10d73ad64..fe717998cf 100644
--- a/tika-ml/tika-ml-junkdetect/pom.xml
+++ b/tika-ml/tika-ml-junkdetect/pom.xml
@@ -61,6 +61,18 @@
</dependency>
<!-- Test dependencies -->
+ <!--
+ tika-serialization is test-scope only because the one consumer
+ (BuildJunkAugmentationData) lives in src/test/java — it's a corpus-prep
+ tool, not part of the runtime detector. Keeps the production classpath of
+ tika-ml-junkdetect free of the serialization dep.
+ -->
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-serialization</artifactId>
+ <version>${revision}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/V7Tables.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/BigramTables.java
similarity index 86%
rename from tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/V7Tables.java
rename to tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/BigramTables.java
index 93a82640ca..5c7e738290 100644
--- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/V7Tables.java
+++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/BigramTables.java
@@ -23,13 +23,10 @@ import java.nio.ByteBuffer;
import java.nio.ByteOrder;
/**
- * Carrier for one script's v7 F1 tables.
+ * Carrier for one script's codepoint-bigram tables.
*
- * <p>The v6 design used a single global codepoint-bigram hash + Bloom
- * filter shared across all scripts. We measured that this ceiling
- * limits accuracy: enlarging one script's training data (e.g. HAN) hurts
- * the other scripts' z-scores because they share the global hash. v7
- * gives each script its own pair of tables.
+ * <p>Each script has its own pair of tables so that enlarging one script's
+ * training data does not perturb the others' z-scores.
*
* <p>Per-script layout:
*
@@ -56,20 +53,19 @@ import java.nio.ByteOrder;
* absent codepoints don't accidentally score above legitimately-rare
* ones.
* <li>{@code backoffAlpha} — multiplier on the unigram-backoff
- * independence sum, copied from v6.
+ * independence sum.
* </ul>
*
* <p>Membership semantics: no Bloom filter. The empty-slot sentinel is
* the membership oracle — a pair is "seen" iff binary-search finds both
* codepoints in the index AND a probe sequence hits a matching key before
- * an empty slot. Lookups are therefore exact; there is no false-positive
- * backoff path as there is in v6.
+ * an empty slot. Lookups are therefore exact.
*
* <p>Fields are package-private so the
* {@link org.apache.tika.ml.junkdetect.tools.TrainJunkModel} trainer can
* construct instances directly without going through accessors.
*/
-public final class V7Tables {
+public final class BigramTables {
/** Reserved value in {@link #bigramKeys} marking an unoccupied slot. */
public static final int EMPTY_KEY = -1;
@@ -85,13 +81,13 @@ public final class V7Tables {
final float unigramFallbackLogProb;
final float backoffAlpha;
- public V7Tables(int[] codepointIndex,
- int[] bigramKeys, byte[] bigramValues,
- byte[] unigramTable,
- float bigramQuantMin, float bigramQuantMax,
- float unigramQuantMin, float unigramQuantMax,
- float unigramFallbackLogProb,
- float backoffAlpha) {
+ public BigramTables(int[] codepointIndex,
+ int[] bigramKeys, byte[] bigramValues,
+ byte[] unigramTable,
+ float bigramQuantMin, float bigramQuantMax,
+ float unigramQuantMin, float unigramQuantMax,
+ float unigramFallbackLogProb,
+ float backoffAlpha) {
if (bigramKeys.length != bigramValues.length) {
throw new IllegalArgumentException(
"bigramKeys and bigramValues must have equal length: "
@@ -115,7 +111,7 @@ public final class V7Tables {
}
/**
- * Serialises this script's F1 tables. Read back via
+ * Serialises this script's bigram tables. Read back via
* {@link #readFrom(DataInputStream)}.
*/
public void writeTo(DataOutputStream dos) throws IOException {
@@ -146,7 +142,7 @@ public final class V7Tables {
}
/** Inverse of {@link #writeTo(DataOutputStream)}. */
- public static V7Tables readFrom(DataInputStream dis) throws IOException {
+ public static BigramTables readFrom(DataInputStream dis) throws IOException {
float backoffAlpha = dis.readFloat();
int cpCount = dis.readInt();
@@ -167,7 +163,7 @@ public final class V7Tables {
float uFallback = dis.readFloat();
byte[] unigramTable = dis.readNBytes(cpCount);
- return new V7Tables(codepoints, keys, values, unigramTable,
+ return new BigramTables(codepoints, keys, values, unigramTable,
bMin, bMax, uMin, uMax, uFallback, backoffAlpha);
}
diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/HtmlContentCleaner.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/HtmlContentCleaner.java
new file mode 100644
index 0000000000..e531730bce
--- /dev/null
+++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/HtmlContentCleaner.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.junkdetect;
+
+import java.nio.charset.StandardCharsets;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.tika.ml.chardetect.HtmlByteStripper;
+
+/**
+ * Junk-detection HTML→text cleaning: strip tags, then <em>expand</em> entities
+ * to codepoints. Called by both {@code TrainJunkModel} and {@link
+ * JunkFilterEncodingDetector} so training and inference prepare text identically
+ * (no drift).
+ *
+ * <p>Unlike charset detection's {@link HtmlByteStripper#stripTagsAndEntities}
+ * (which <em>drops</em> entities as charset-neutral ASCII noise), junk detection
+ * expands them: the resulting codepoints — cross-script under a wrong decoding —
+ * are what expose mojibake.
+ */
+public final class HtmlContentCleaner {
+
+ private static final Pattern ENTITY_DEC = Pattern.compile("&#(\\d{1,7});");
+ private static final Pattern ENTITY_HEX = Pattern.compile("&#[xX]([0-9a-fA-F]{1,6});");
+ private static final Pattern ENTITY_NAMED =
+ Pattern.compile("&(amp|lt|gt|quot|apos|nbsp|copy|reg);");
+
+ private HtmlContentCleaner() {
+ }
+
+ /**
+ * Strip HTML tags (entities preserved through the strip), then expand
+ * entities to real codepoints. No-op-ish on plain text (no tags/entities).
+ */
+ public static String clean(String s) {
+ if (s == null || s.isEmpty()) {
+ return s;
+ }
+ // Tag-strip via the byte stripper on the string's UTF-8 form; keeps
+ // entities (stripTags, not stripTagsAndEntities) for the expand below.
+ byte[] u = s.getBytes(StandardCharsets.UTF_8);
+ byte[] dst = new byte[u.length];
+ HtmlByteStripper.Result r = HtmlByteStripper.stripTags(u, 0, u.length, dst, 0);
+ String tagless = (r.tagCount > 0 && r.length > 0)
+ ? new String(dst, 0, r.length, StandardCharsets.UTF_8)
+ : s;
+ return expandHtmlEntities(tagless);
+ }
+
+ /**
+ * Expand HTML entities to codepoints. Numeric refs ({@code ©},
+ * {@code ©}) are fully decoded; a small set of common named entities
+ * is mapped; other named entities pass through literally.
+ */
+ static String expandHtmlEntities(String s) {
+ s = ENTITY_DEC.matcher(s).replaceAll(mr -> {
+ try {
+ int cp = Integer.parseInt(mr.group(1));
+ if (cp >= 0 && cp <= 0x10FFFF) {
+ return Matcher.quoteReplacement(new String(Character.toChars(cp)));
+ }
+ } catch (NumberFormatException ignored) {
+ // overflow — leave literal
+ }
+ return Matcher.quoteReplacement(mr.group());
+ });
+ s = ENTITY_HEX.matcher(s).replaceAll(mr -> {
+ try {
+ int cp = Integer.parseInt(mr.group(1), 16);
+ if (cp >= 0 && cp <= 0x10FFFF) {
+ return Matcher.quoteReplacement(new String(Character.toChars(cp)));
+ }
+ } catch (NumberFormatException ignored) {
+ // overflow — leave literal
+ }
+ return Matcher.quoteReplacement(mr.group());
+ });
+ s = ENTITY_NAMED.matcher(s).replaceAll(mr -> {
+ switch (mr.group(1)) {
+ case "amp": return "&";
+ case "lt": return "<";
+ case "gt": return ">";
+ case "quot": return "\"";
+ case "apos": return "'";
+ case "nbsp": return " ";
+ case "copy": return "©";
+ case "reg": return "®";
+ default: return Matcher.quoteReplacement(mr.group());
+ }
+ });
+ return s;
+ }
+}
diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java
index 5635f6f168..c231494d60 100644
--- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java
+++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java
@@ -24,11 +24,9 @@ import java.nio.ByteOrder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
-import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.zip.GZIPInputStream;
@@ -41,29 +39,30 @@ import org.apache.tika.quality.TextQualityScore;
* Language-agnostic text quality scorer. Discriminates clean UTF-8 text from
* mojibake, reversed text, wrong-codec decodings, and other corruption forms.
*
- * <p>Scoring combines four features:
+ * <p>Scoring combines several features:
* <ol>
- * <li><b>Codepoint-bigram log-probability (F1)</b> — global hashed table
- * indexed by FNV-1a(cp_a, cp_b, seed) into {@code bigramBuckets} cells.
- * A Bloom filter records seen pairs; unseen pairs fall back to a
- * hashed-unigram independence-assumption score
- * {@code α * (log P(cp_a) + log P(cp_b))}.</li>
- * <li><b>Unicode named-block transition log-probability (F2)</b> —
- * per-script N×N table over {@link Character.UnicodeBlock} values.</li>
- * <li><b>Control-byte fraction (F3)</b> — fraction of bytes in control
+ * <li><b>Codepoint-bigram log-probability (z1)</b> — every bigram is
+ * bucketed to its script ({@link #forEachScriptBigram}) and scored
+ * against that script's open-addressed bigram table; unseen pairs fall
+ * back to a unigram independence-assumption score
+ * {@code α * (log P(cp_a) + log P(cp_b))}. This is the only per-script
+ * feature.</li>
+ * <li><b>Unicode block-transition log-probability (z2)</b> — a single
+ * global table over {@link Character.UnicodeBlock} values.</li>
+ * <li><b>Control-byte fraction (z3)</b> — fraction of bytes in control
* ranges [0x01–0x08, 0x0B, 0x0C, 0x0E–0x1F, 0x7F].</li>
- * <li><b>Global script-transition log-probability (F4)</b> — single
- * transition table over raw {@link Character.UnicodeScript} values,
- * capturing document-level cross-script anomalies.</li>
+ * <li><b>Script-transition log-probability (z4)</b> — a single global
+ * table over raw {@link Character.UnicodeScript} values.</li>
+ * <li>Document-level quality features z5..z9 (letter-adjacent-to-mark,
+ * replacement ratio, script density, script coherence, script
+ * alternation).</li>
* </ol>
*
- * <p>All features are calibrated per-script (mu/sigma) on held-out dev text
- * so their z-scores are on a common scale. z-scores are combined by a
- * per-script linear classifier:
- * {@code logit = w1*z1 + w2*z2 + w3*z3 + w4*z4 + bias}, where weights are
- * fit on clean vs. corrupted dev windows. Natural junk threshold is 0
- * (positive logit = clean); use negative thresholds for conservative
- * detection.</p>
+ * <p>Per-script z1 is calibrated (mu/sigma) on held-out dev text; z2..z9 are
+ * global. The z-scores are combined by a single global linear combiner:
+ * {@code logit = w1*z1 + ... + w9*z9 + bias}, fit on clean vs. corrupted
+ * windows. Natural junk threshold is 0 (positive logit = clean); use
+ * negative thresholds for conservative detection.</p>
*
* <p>Model file format: a single binary spec (see {@link #load(InputStream)}
* javadoc). No backwards-compat fallback to older formats — the loader
@@ -91,54 +90,88 @@ public final class JunkDetector implements TextQualityDetector {
"org/apache/tika/ml/junkdetect/junkdetect.bin";
static final String MAGIC = "JUNKDET1";
- /** Sole supported file-format version. Mismatch is a hard error. */
- static final int VERSION = 7;
+ /** Sole supported file-format version. Mismatch is a hard error —
+ * prior versions live in git history and are not loadable by this
+ * build. We deliberately don't keep dual-version paths so it's
+ * impossible to confuse model versions. */
+ // Bucket-by-script model. z1 (codepoint-bigram LM) is the only per-script
+ // feature, scored by bucketing every bigram to its script (digits skipped,
+ // COMMON glue folded into the adjacent script). z2 (block-transition) and
+ // z3 (control-byte) are single GLOBAL document-level features; the combiner
+ // is one global weight vector. Older model files are structurally
+ // incompatible and rejected at load.
+ public static final int VERSION = 15;
// Feature 1 — per-script open-addressed codepoint-bigram tables.
// No global Bloom: empty-slot is the membership oracle.
- private final Map<String, V7Tables> f1TablesByScript;
+ private final Map<String, BigramTables> f1TablesByScript;
- /** Per-script F1 calibration on the codepoint-hash mean log-prob. */
+ /** Per-script z1 calibration {mu, sigma} on the bucket mean log-prob.
+ * z1 is the ONLY per-script feature. */
private final Map<String, float[]> calibrations; // script → float[2] {mu, sigma}
- // Feature 2 — per-script block transition. Block bucketing uses the
- // JVM-independent {@link UnicodeBlockRanges} static table; table size
- // per script is {@code bucketCount()²} floats.
- private final Map<String, float[]> blockTables;
- private final Map<String, float[]> blockCalibrations;
+ // Feature 2 — single GLOBAL block-transition table.
+ // int16-quantized; dequantize via {@code min + (s/65535) * (max - min)}.
+ private final short[] blockTable;
+ private final float[] blockTableQuant; // {min, max}
+ private final float[] blockCalibration; // {mu, sigma}
- // Feature 3 — per-script control-byte fraction calibration
- private final Map<String, float[]> controlCalibrations;
+ // Feature 3 — single GLOBAL control-byte fraction calibration {mu, sigma}.
+ private final float[] controlCalibration;
- // Feature 4 — single global script-transition table
- private final float[] scriptTransitionTable;
+ // Feature 4 — single global script-transition table (int16 quantized).
+ private final short[] scriptTransitionTable;
+ private final float[] scriptTransitionTableQuant; // {min, max}
private final float[] scriptTransitionCalibration;
private final Map<String, Integer> scriptBucketIndex;
private final int numScriptBuckets;
- // Per-script linear classifier: float[numFeatures+1] = {w1, ..., wN, bias}.
- private final Map<String, float[]> classifierWeights;
+ // Single GLOBAL linear combiner: float[numFeatures+1] = {w1, ..., wN, bias}.
+ private final float[] combinerWeights;
+
+ /** Document-level z5 calibration {mu, sigma}. */
+ private final float[] z5Calibration;
+ /** Document-level z6 calibration {mu, sigma}. */
+ private final float[] z6Calibration;
+ /** Document-level z9 calibration {mu, sigma}. z9 = scriptAlternationRatio:
+ * transitions between dominant and foreign script, normalized by max
+ * possible transitions given the counts. Length- and proportion-
+ * invariant. Catches the mojibake-of-Latin-as-CJK pattern: scattered
+ * singleton Han chars in Latin text score near 1.0 (max alternation),
+ * while legitimate mixed-script (English with embedded Chinese phrase)
+ * scores low because the foreign script clumps together. */
+ private final float[] z9Calibration;
private JunkDetector(Map<String, float[]> calibrations,
- Map<String, float[]> blockTables,
- Map<String, float[]> blockCalibrations,
- Map<String, float[]> controlCalibrations,
- Map<String, float[]> classifierWeights,
- float[] scriptTransitionTable,
+ short[] blockTable,
+ float[] blockTableQuant,
+ float[] blockCalibration,
+ float[] controlCalibration,
+ float[] combinerWeights,
+ short[] scriptTransitionTable,
+ float[] scriptTransitionTableQuant,
float[] scriptTransitionCalibration,
Map<String, Integer> scriptBucketIndex,
int numScriptBuckets,
- Map<String, V7Tables> f1TablesByScript) {
+ Map<String, BigramTables> f1TablesByScript,
+ float[] z5Calibration,
+ float[] z6Calibration,
+ float[] z9Calibration) {
this.calibrations = Collections.unmodifiableMap(calibrations);
- this.blockTables = Collections.unmodifiableMap(blockTables);
- this.blockCalibrations = Collections.unmodifiableMap(blockCalibrations);
- this.controlCalibrations = Collections.unmodifiableMap(controlCalibrations);
- this.classifierWeights = Collections.unmodifiableMap(classifierWeights);
+ this.blockTable = blockTable;
+ this.blockTableQuant = blockTableQuant;
+ this.blockCalibration = blockCalibration;
+ this.controlCalibration = controlCalibration;
+ this.combinerWeights = combinerWeights;
this.scriptTransitionTable = scriptTransitionTable;
+ this.scriptTransitionTableQuant = scriptTransitionTableQuant;
this.scriptTransitionCalibration = scriptTransitionCalibration;
this.scriptBucketIndex = Collections.unmodifiableMap(scriptBucketIndex);
this.numScriptBuckets = numScriptBuckets;
this.f1TablesByScript = Collections.unmodifiableMap(f1TablesByScript);
+ this.z5Calibration = z5Calibration;
+ this.z6Calibration = z6Calibration;
+ this.z9Calibration = z9Calibration;
}
// -----------------------------------------------------------------------
@@ -198,7 +231,7 @@ public final class JunkDetector implements TextQualityDetector {
* <p>File-format layout (gzipped):
* <pre>
* [8 bytes] magic "JUNKDET1" (ASCII)
- * [1 byte] version (= 7)
+ * [1 byte] version (= {@value #VERSION})
* [4 bytes] num_scripts (int BE)
* [1 byte] block_scheme_version (must equal
* {@link UnicodeBlockRanges#SCHEME_VERSION})
@@ -214,7 +247,7 @@ public final class JunkDetector implements TextQualityDetector {
* [name bytes] script name (UTF-8)
* [4 bytes] mu1 (F1 calibration, codepoint-bigram mean log-prob)
* [4 bytes] sigma1
- * // V7 F1 tables for this script — see {@link V7Tables#writeTo}
+ * // bigram tables for this script — see {@link BigramTables#writeTo}
* [4 bytes] backoff_alpha (float32 BE)
* [4 bytes] codepoint_count
* [codepoint_count × 4 bytes] codepoint index (sorted, ascending)
@@ -222,14 +255,14 @@ public final class JunkDetector implements TextQualityDetector {
* [4 bytes] bigram_quant_min (float32 BE)
* [4 bytes] bigram_quant_max (float32 BE)
* [bigram_slots × 4 bytes] bigram open-addressing keys
- * ((idxA<<16)|idxB, or {@link V7Tables#EMPTY_KEY})
+ * ((idxA<<16)|idxB, or {@link BigramTables#EMPTY_KEY})
* [bigram_slots bytes] bigram values (8-bit quantized log-probs)
* [4 bytes] unigram_quant_min (float32 BE)
* [4 bytes] unigram_quant_max (float32 BE)
* [4 bytes] unigram_fallback_log_prob (float32 BE; used for
* codepoints not in index)
* [codepoint_count bytes] unigram values (8-bit quantized log-probs)
- * // F2/F3/classifier (unchanged from v6 layout)
+ * // F2/F3/classifier
* [4 bytes] mu2 (F2 calibration)
* [4 bytes] sigma2
* [block_N² × 4 bytes] block-transition log-prob table (F2)
@@ -273,7 +306,7 @@ public final class JunkDetector implements TextQualityDetector {
}
int blockN = UnicodeBlockRanges.bucketCount();
- // Global script-transition section
+ // z4 — global script-transition section (int16-quantized table).
int numScriptBuckets = dis.readUnsignedByte();
Map<String, Integer> scriptBucketIndex = new LinkedHashMap<>(numScriptBuckets * 2);
for (int i = 0; i < numScriptBuckets; i++) {
@@ -281,45 +314,63 @@ public final class JunkDetector implements TextQualityDetector {
String bucketName = new String(dis.readNBytes(nameLen), StandardCharsets.UTF_8);
scriptBucketIndex.put(bucketName, i);
}
- float[] scriptTransitionTable = readFloatTable(dis, numScriptBuckets * numScriptBuckets);
+ float scriptTransMin = dis.readFloat();
+ float scriptTransMax = dis.readFloat();
+ short[] scriptTransitionTable = readShortTable(dis, numScriptBuckets * numScriptBuckets);
+ float[] scriptTransitionTableQuant = new float[]{scriptTransMin, scriptTransMax};
float[] scriptTransitionCalibration = new float[]{dis.readFloat(), dis.readFloat()};
- Map<String, V7Tables> f1TablesByScript = new HashMap<>(numScripts * 2);
- Map<String, float[]> calibrations = new HashMap<>(numScripts * 2);
- Map<String, float[]> blockTables = new HashMap<>(numScripts * 2);
- Map<String, float[]> blockCalibrations = new HashMap<>(numScripts * 2);
- Map<String, float[]> controlCalibrations = new HashMap<>(numScripts * 2);
- Map<String, float[]> classifierWeights = new HashMap<>(numScripts * 2);
+ // z2 — single GLOBAL block-transition table (int16-quantized).
+ float blockMin = dis.readFloat();
+ float blockMax = dis.readFloat();
+ short[] blockTable = readShortTable(dis, blockN * blockN);
+ float[] blockTableQuant = new float[]{blockMin, blockMax};
+ float[] blockCalibration = new float[]{dis.readFloat(), dis.readFloat()};
+
+ // z3 — single GLOBAL control-byte calibration.
+ float[] controlCalibration = new float[]{dis.readFloat(), dis.readFloat()};
+
+ // Document-level calibrations: z5 (letter-adjacent-to-mark),
+ // z6 (replacement-char), z9 (script-alternation).
+ float[] z5Calibration = new float[]{dis.readFloat(), dis.readFloat()};
+ float[] z6Calibration = new float[]{dis.readFloat(), dis.readFloat()};
+ float[] z9Calibration = new float[]{dis.readFloat(), dis.readFloat()};
+
+ // Single GLOBAL combiner weights {w1..wN, bias}.
+ int numFeatures = dis.readUnsignedByte();
+ float[] combinerWeights = new float[numFeatures + 1];
+ for (int j = 0; j <= numFeatures; j++) {
+ combinerWeights[j] = dis.readFloat();
+ }
+ // Per-script section: only z1 calibration + the bigram tables.
+ Map<String, BigramTables> f1TablesByScript = new HashMap<>(numScripts * 2);
+ Map<String, float[]> calibrations = new HashMap<>(numScripts * 2);
for (int s = 0; s < numScripts; s++) {
int nameLen = dis.readUnsignedShort();
String script = new String(dis.readNBytes(nameLen), StandardCharsets.UTF_8);
-
calibrations.put(script, new float[]{dis.readFloat(), dis.readFloat()});
-
- // Per-script V7 F1 tables.
- f1TablesByScript.put(script, V7Tables.readFrom(dis));
-
- blockCalibrations.put(script, new float[]{dis.readFloat(), dis.readFloat()});
- blockTables.put(script, readFloatTable(dis, blockN * blockN));
- controlCalibrations.put(script, new float[]{dis.readFloat(), dis.readFloat()});
-
- int numFeatures = dis.readUnsignedByte();
- float[] weights = new float[numFeatures + 1];
- for (int j = 0; j <= numFeatures; j++) {
- weights[j] = dis.readFloat();
- }
- classifierWeights.put(script, weights);
+ f1TablesByScript.put(script, BigramTables.readFrom(dis));
}
return new JunkDetector(calibrations,
- blockTables, blockCalibrations,
- controlCalibrations, classifierWeights,
- scriptTransitionTable, scriptTransitionCalibration,
- scriptBucketIndex, numScriptBuckets, f1TablesByScript);
+ blockTable, blockTableQuant, blockCalibration,
+ controlCalibration, combinerWeights,
+ scriptTransitionTable, scriptTransitionTableQuant,
+ scriptTransitionCalibration,
+ scriptBucketIndex, numScriptBuckets, f1TablesByScript,
+ z5Calibration, z6Calibration, z9Calibration);
}
}
+ /** Read {@code size} big-endian int16 values as a short[]. */
+ private static short[] readShortTable(DataInputStream dis, int size) throws IOException {
+ byte[] raw = dis.readNBytes(size * 2);
+ short[] out = new short[size];
+ ByteBuffer.wrap(raw).order(ByteOrder.BIG_ENDIAN).asShortBuffer().get(out);
+ return out;
+ }
+
private static float[] readFloatTable(DataInputStream dis, int size) throws IOException {
byte[] tableBytes = dis.readNBytes(size * 4);
float[] table = new float[size];
@@ -385,7 +436,8 @@ public final class JunkDetector implements TextQualityDetector {
return calibrations.keySet();
}
- /** Returns the file-format version of the loaded model. */
+ /** Returns the file-format version of the loaded model
+ * (always {@link #VERSION}; mismatches are rejected at load time). */
public int getModelVersion() {
return VERSION;
}
@@ -394,137 +446,141 @@ public final class JunkDetector implements TextQualityDetector {
// Internal scoring
// -----------------------------------------------------------------------
- private TextQualityScore scoreText(String text) {
- List<ScriptRun> runs = buildScriptRuns(text);
-
- // Global z4: script-transition feature over the whole input string.
- // Computed before chunking because it captures document-level script mixing.
- float z4 = computeScriptTransitionZ(text);
-
- // Score each run against its own model; aggregate weighted by byte count.
- float totalBytes = 0;
- float weightedLogit = 0;
- String dominantScript = null;
- int maxBytes = 0;
- int totalBigramCount = 0;
- float[] dominantCal1 = null;
-
- for (ScriptRun run : runs) {
- if (!calibrations.containsKey(run.script)) {
- continue; // skip scripts not in model; treat as neutral, not junk
+ /** Aggregated document features under the bucket-by-script model. */
+ private static final class Agg {
+ float z1, z2, z3, z4, z5, z6, z7, z8, z9, logit;
+ String dominantScript; // null => no scoreable script (NONE fallback)
+ int totalBigrams;
+ float[] dominantCal1;
+ }
+
+ /**
+ * Computes the document feature vector under the bucket-by-script model:
+ * every bigram is bucketed to its script ({@link #forEachScriptBigram}),
+ * each script's mean log-prob is calibrated to {@code z1_S=(mean-mu)/sigma},
+ * and the per-script z1's are aggregated count-weighted into a single doc
+ * {@code z1}. z2..z9 are document-level globals. A single global combiner
+ * produces the logit. No runs, no sentinels, no per-script z2/z3.
+ */
+ private Agg aggregate(String text) {
+ // NFC-normalize so inference matches the trainer's tally.
+ text = java.text.Normalizer.normalize(text, java.text.Normalizer.Form.NFC);
+ int[] cps = text.codePoints().toArray();
+
+ Map<String, double[]> buckets = new HashMap<>(); // script -> {sumLogP, count}
+ forEachScriptBigram(cps, (script, a, b) -> {
+ if (!calibrations.containsKey(script)) {
+ return;
+ }
+ BigramTables t = f1TablesByScript.get(script);
+ if (t == null) {
+ return;
}
- byte[] runUtf8 = run.text.getBytes(StandardCharsets.UTF_8);
- // Skip if too short to form a bigram by either metric. A single
- // CJK char is 3 UTF-8 bytes (passes the byte filter) but 1 UTF-16
- // unit, and computeF1MeanLogP filters by text.length() < 2 and
- // returns NaN — which would poison the weighted sum here.
- if (runUtf8.length < 2 || run.text.length() < 2) {
+ double lp = computeF1MeanLogP(new int[]{a, b}, t);
+ if (Double.isNaN(lp)) {
+ return;
+ }
+ double[] bk = buckets.computeIfAbsent(script, k -> new double[2]);
+ bk[0] += lp;
+ bk[1] += 1;
+ });
+
+ Agg agg = new Agg();
+ double weightedZ1 = 0;
+ long z1Count = 0;
+ long maxCount = 0;
+ for (Map.Entry<String, double[]> e : buckets.entrySet()) {
+ long cnt = (long) e.getValue()[1];
+ if (cnt == 0) {
continue;
}
- float logit = scoreChunk(runUtf8, run.text, run.script, z4);
- int n = runUtf8.length;
- weightedLogit += logit * n;
- totalBytes += n;
- totalBigramCount += n - 1;
- if (n > maxBytes) {
- maxBytes = n;
- dominantScript = run.script;
- dominantCal1 = calibrations.get(run.script);
+ float[] cal = calibrations.get(e.getKey());
+ double mean = e.getValue()[0] / cnt;
+ double z1s = (mean - cal[0]) / cal[1];
+ weightedZ1 += z1s * cnt;
+ z1Count += cnt;
+ if (cnt > maxCount) {
+ maxCount = cnt;
+ agg.dominantScript = e.getKey();
+ agg.dominantCal1 = cal;
}
}
- if (totalBytes == 0 || dominantScript == null) {
- String label = runs.isEmpty() ? "LATIN" : runs.get(0).script;
- return unknownScore(label);
+ // Document-level features (computed once).
+ agg.z2 = computeZ2BlockTransitionQuantized(text, blockTable,
+ blockTableQuant, blockCalibration);
+ agg.z3 = computeZ3ControlByte(text.getBytes(StandardCharsets.UTF_8),
+ controlCalibration);
+ agg.z4 = computeScriptTransitionZ(text);
+ agg.z5 = computeZ5LetterAdjacentToMarkRatio(text);
+ agg.z6 = computeZ6ReplacementRatio(text);
+ float z7 = (float) TextQualityFeatures.scriptDensity(text);
+ agg.z7 = Float.isNaN(z7) ? 0f : z7;
+ double rawFrag = TextQualityFeatures.scriptFragmentation(text);
+ agg.z8 = Double.isNaN(rawFrag) ? 1f : 1f - (float) rawFrag;
+ agg.z9 = computeZ9AlternationRatio(text);
+ agg.totalBigrams = (int) z1Count;
+
+ if (z1Count == 0 || agg.dominantScript == null) {
+ // No scoreable script — doc-level fallback (same anchors as before):
+ // density=0 -> very negative; density=1 coherence=1 -> positive
+ // (unmodeled coherent script); density=1 coherence=0 -> very negative.
+ agg.dominantScript = null;
+ agg.z1 = Float.NaN;
+ agg.logit = -7f + 4f * agg.z7 + 6f * agg.z8;
+ return agg;
}
+ agg.z1 = (float) (weightedZ1 / z1Count);
+ agg.logit = combineLogit(agg.z1, agg.z2, agg.z3, agg.z4,
+ agg.z5, agg.z6, agg.z7, agg.z8, agg.z9);
+ return agg;
+ }
- float zScore = weightedLogit / totalBytes;
-
- float uncertainty = (dominantCal1 != null && totalBigramCount > 0)
- ? (float) (1.96 * dominantCal1[1] / Math.sqrt(totalBigramCount)) : 0f;
- float ciLow = zScore - uncertainty;
- float ciHigh = zScore + uncertainty;
- float pClean = (float) (1.0 / (1.0 + Math.exp(-zScore)));
-
- return new TextQualityScore(zScore, pClean, ciLow, ciHigh, dominantScript);
+ private TextQualityScore scoreText(String text) {
+ Agg a = aggregate(text);
+ float pClean = (float) (1.0 / (1.0 + Math.exp(-a.logit)));
+ if (a.dominantScript == null) {
+ return new TextQualityScore(a.logit, pClean, a.logit, a.logit, "NONE");
+ }
+ float uncertainty = (a.dominantCal1 != null && a.totalBigrams > 0)
+ ? (float) (1.96 * a.dominantCal1[1] / Math.sqrt(a.totalBigrams)) : 0f;
+ return new TextQualityScore(a.logit, pClean,
+ a.logit - uncertainty, a.logit + uncertainty, a.dominantScript);
}
/**
- * Diagnostic — exposes per-feature z-scores and classifier weights. Same
- * chunking and aggregation as {@link #score(String)}, but returns the
- * intermediate signals individually for analysis or for hybrid models
- * that want to substitute one feature with an externally-computed value.
- *
- * <p>Aggregation: per-chunk z1/z2/z3 and per-chunk logit are byte-count-
- * weighted across script-homogeneous chunks. z4 is a global signal
- * (already document-level). {@code dominantScript} and
- * {@code classifierWeights} refer to the script run with the most bytes.
+ * Diagnostic — exposes the per-feature z-scores, the global combiner
+ * weights, and the dominant script. Same math as {@link #score(String)}.
*/
public FeatureComponents scoreWithFeatureComponents(String text) {
if (text == null || text.isEmpty()) {
return new FeatureComponents(Float.NaN, Float.NaN, Float.NaN,
+ Float.NaN, Float.NaN, Float.NaN, Float.NaN, Float.NaN,
Float.NaN, Float.NaN, "UNKNOWN", null, 0);
}
- List<ScriptRun> runs = buildScriptRuns(text);
- float z4 = computeScriptTransitionZ(text);
-
- float totalBytes = 0;
- float weightedZ1 = 0;
- float weightedZ2 = 0;
- float weightedZ3 = 0;
- float weightedLogit = 0;
- String dominantScript = null;
- int maxBytes = 0;
-
- for (ScriptRun run : runs) {
- if (!calibrations.containsKey(run.script)) {
- continue;
- }
- byte[] runUtf8 = run.text.getBytes(StandardCharsets.UTF_8);
- if (runUtf8.length < 2 || run.text.length() < 2) {
- continue; // see scoreText: paired filter avoids NaN poisoning
- }
- float[] zs = computeChunkZs(runUtf8, run.text, run.script);
- float chunkLogit = combineLogit(zs[0], zs[1], zs[2], z4, run.script);
- int n = runUtf8.length;
- weightedZ1 += zs[0] * n;
- weightedZ2 += zs[1] * n;
- weightedZ3 += zs[2] * n;
- weightedLogit += chunkLogit * n;
- totalBytes += n;
- if (n > maxBytes) {
- maxBytes = n;
- dominantScript = run.script;
- }
- }
-
- if (totalBytes == 0 || dominantScript == null) {
- return new FeatureComponents(Float.NaN, Float.NaN, Float.NaN, z4,
- Float.NaN, runs.isEmpty() ? "UNKNOWN" : runs.get(0).script,
- null, 0);
+ Agg a = aggregate(text);
+ if (a.dominantScript == null) {
+ return new FeatureComponents(Float.NaN, Float.NaN, Float.NaN,
+ a.z4, a.z5, a.z6, a.z7, a.z8, a.z9, a.logit, "NONE", null, 0);
}
-
- float[] cw = classifierWeights.get(dominantScript);
- return new FeatureComponents(
- weightedZ1 / totalBytes,
- weightedZ2 / totalBytes,
- weightedZ3 / totalBytes,
- z4,
- weightedLogit / totalBytes,
- dominantScript,
- cw,
- (int) totalBytes);
+ return new FeatureComponents(a.z1, a.z2, a.z3, a.z4, a.z5, a.z6, a.z7,
+ a.z8, a.z9, a.logit, a.dominantScript, combinerWeights,
+ a.totalBigrams);
}
/**
* Per-feature z-score breakdown returned by
- * {@link #scoreWithFeatureComponents(String)}. All z-scores are
- * byte-count-weighted aggregates across script-homogeneous chunks
- * except {@code z4}, which is a single document-level value.
+ * {@link #scoreWithFeatureComponents(String)}. z1-z3 are byte-count-
+ * weighted aggregates across script-homogeneous chunks; z4-z6 are
+ * single document-level values.
+ *
+ * <p>z5 (letter-adjacent-to-mark ratio) and z6 (replacement-character
+ * ratio) are document-level features included alongside z1–z4 in the
+ * per-script LR.
*
* <p>{@code classifierWeights} is the per-script linear classifier
- * weight vector {@code {w1, w2, w3, w4, bias}} for the dominant
- * script — useful for hybrid models that recompute the logit after
+ * weight vector {@code {w1, ..., wN, bias}} for the dominant script
+ * — useful for hybrid models that recompute the logit after
* substituting one z-score with an externally-computed value.
*/
public static final class FeatureComponents {
@@ -532,18 +588,29 @@ public final class JunkDetector implements TextQualityDetector {
public final float z2;
public final float z3;
public final float z4;
+ public final float z5;
+ public final float z6;
+ public final float z7;
+ public final float z8;
+ public final float z9;
public final float logit;
public final String dominantScript;
public final float[] classifierWeights;
public final int totalBytes;
FeatureComponents(float z1, float z2, float z3, float z4,
+ float z5, float z6, float z7, float z8, float z9,
float logit, String dominantScript,
float[] classifierWeights, int totalBytes) {
this.z1 = z1;
this.z2 = z2;
this.z3 = z3;
this.z4 = z4;
+ this.z5 = z5;
+ this.z6 = z6;
+ this.z7 = z7;
+ this.z8 = z8;
+ this.z9 = z9;
this.logit = logit;
this.dominantScript = dominantScript;
this.classifierWeights = classifierWeights;
@@ -551,45 +618,106 @@ public final class JunkDetector implements TextQualityDetector {
}
}
+ // -----------------------------------------------------------------------
+ // Global features (computed once per document, like z4)
+ // -----------------------------------------------------------------------
+
/**
- * Scores a single script-homogeneous chunk and returns its logit.
- * Positive = clean, negative = junk. Returns 0 (neutral) if the chunk
- * has no model or is too short.
+ * z5: calibrated letter-adjacent-to-mark ratio. Delegates raw
+ * computation to {@link TextQualityFeatures#letterAdjacentToMarkRatio}
+ * and applies the document-level (mu, sigma) calibration loaded from
+ * the model file. Returns 0 (neutral) when the model has no z5
+ * calibration or when the raw value is NaN.
+ *
+ * <p>Positive z5 = correct decoding of a precomposed-or-decomposed
+ * script (Vietnamese, Indic, Thai, Arabic). Negative z5 = mojibake
+ * of such content as Latin-1.
*/
- private float scoreChunk(byte[] utf8, String text, String script, float z4) {
- if (utf8.length < 2 || !calibrations.containsKey(script)) {
+ public float computeZ5LetterAdjacentToMarkRatio(String text) {
+ double raw = TextQualityFeatures.letterAdjacentToMarkRatio(text);
+ if (Double.isNaN(raw) || z5Calibration == null || z5Calibration[1] <= 0) {
return 0f;
}
- float[] zs = computeChunkZs(utf8, text, script);
- return combineLogit(zs[0], zs[1], zs[2], z4, script);
+ return ((float) raw - z5Calibration[0]) / z5Calibration[1];
}
/**
- * Computes per-feature z-scores {z1, z2, z3} for a single script-
- * homogeneous chunk. Shared between {@link #scoreChunk} and
- * {@link #scoreWithFeatureComponents}, and used at training time
- * via the public {@code computeZ2/3/4...} static helpers so
- * training and inference share the same math.
+ * z6: calibrated replacement-character ratio. Direct decode-failure
+ * signal — fraction of codepoints that are U+FFFD. Higher raw value
+ * = more decode failure = junkier; but the calibration centers on the
+ * training distribution, so negative z6 = junkier than typical.
+ *
+ * <p>Returns 0 (neutral) when no calibration available.
*/
- private float[] computeChunkZs(byte[] utf8, String text, String script) {
- // Feature 1: per-script codepoint-bigram, calibrated per-script
- V7Tables tables = f1TablesByScript.get(script);
- float meanF1LogProb = computeCodepointF1MeanLogP(text, tables);
- float[] cal1 = calibrations.get(script);
- float z1 = (meanF1LogProb - cal1[0]) / cal1[1];
+ public float computeZ6ReplacementRatio(String text) {
+ double raw = TextQualityFeatures.replacementRatio(text);
+ if (Double.isNaN(raw) || z6Calibration == null || z6Calibration[1] <= 0) {
+ return 0f;
+ }
+ // Flip sign: higher replacement = lower quality, so feature is
+ // (mu - raw) / sigma so a clean decode → positive z6.
+ return (z6Calibration[0] - (float) raw) / z6Calibration[1];
+ }
- float z2 = computeZ2BlockTransition(text,
- blockTables.get(script), blockCalibrations.get(script));
- float z3 = computeZ3ControlByte(utf8, controlCalibrations.get(script));
- return new float[]{z1, z2, z3};
+ /**
+ * z9: calibrated script-alternation ratio. Catches the mojibake-of-
+ * Latin-as-CJK pattern where every accent becomes a singleton Han
+ * char scattered through Latin text (high alternation = max value).
+ * Length- and proportion-invariant by construction. Sign flipped so
+ * clean (low alternation) → positive z9 and mojibake (high
+ * alternation) → negative.
+ */
+ public float computeZ9AlternationRatio(String text) {
+ double raw = TextQualityFeatures.scriptAlternationRatio(text);
+ if (Double.isNaN(raw) || z9Calibration == null || z9Calibration[1] <= 0) {
+ return 0f;
+ }
+ // Higher alternation = junkier; (mu - raw) / sigma so clean text → positive z9.
+ return (z9Calibration[0] - (float) raw) / z9Calibration[1];
}
- private static float computeCodepointF1MeanLogP(String text, V7Tables tables) {
- if (tables == null) return Float.NaN;
- double v = computeF1MeanLogP(text, tables);
- return Double.isNaN(v) ? Float.NaN : (float) v;
+
+ /**
+ * Inference-side z2 lookup against an int16-quantized block table.
+ * Mirrors {@link #computeZ2BlockTransition}(float[]) but reads from
+ * the quantized {@code short[]} table with per-table {min, max}
+ * dequant params. Per-bigram
+ * dequantize is {@code min + (s/65535) * (max - min)} where s is
+ * the unsigned 16-bit value. 65536 levels keep ~0.0002 nats/level
+ * resolution — essentially lossless vs the float32 form for our
+ * log-prob range.
+ */
+ private static float computeZ2BlockTransitionQuantized(String text,
+ short[] blockTable,
+ float[] quant,
+ float[] blockCal) {
+ if (blockTable == null || quant == null || blockCal == null || text.length() < 2) {
+ return 0f;
+ }
+ int blockN = UnicodeBlockRanges.bucketCount();
+ float min = quant[0];
+ float scale = (quant[1] - min) / 65535f;
+ int prev = -1;
+ double sum = 0;
+ int count = 0;
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ int blockId = UnicodeBlockRanges.bucketOf(cp);
+ if (prev >= 0) {
+ int s = blockTable[prev * blockN + blockId] & 0xFFFF;
+ sum += min + s * scale;
+ count++;
+ }
+ prev = blockId;
+ i += Character.charCount(cp);
+ }
+ if (count == 0) {
+ return 0f;
+ }
+ return ((float) (sum / count) - blockCal[0]) / blockCal[1];
}
+
/**
* Feature 2 — calibrated z-score for block-transition mean log-prob on
* one text window. Returns 0 if the window has fewer than two
@@ -698,18 +826,32 @@ public final class JunkDetector implements TextQualityDetector {
}
/**
- * Combines per-feature z-scores via the per-script linear classifier.
- * Fallback (when no classifier weights stored): equal-weight average.
+ * Combines per-feature z-scores via the global linear classifier.
+ * Fallback (when no classifier weights stored): equal-weight average of
+ * the four bigram-/transition-based features (z1-z4).
+ *
+ * <p>The classifier has 9 weights + bias (nFeat == 9) —
+ * z1 (bigram), z2 (block transitions), z3 (control bytes),
+ * z4 (script transitions), z5 (letter-adjacent-to-mark),
+ * z6 (replacement ratio), z7 (script density), z8 (script coherence),
+ * z9 (script-alternation ratio).
*/
- private float combineLogit(float z1, float z2, float z3, float z4, String script) {
- float[] cw = classifierWeights.get(script);
- if (cw != null) {
+ private float combineLogit(float z1, float z2, float z3, float z4,
+ float z5, float z6, float z7, float z8,
+ float z9) {
+ float[] cw = combinerWeights;
+ if (cw != null && cw.length >= 1) {
int nFeat = cw.length - 1; // bias is last
float logit = cw[nFeat]; // bias
if (nFeat >= 1) logit += cw[0] * z1;
if (nFeat >= 2) logit += cw[1] * z2;
if (nFeat >= 3) logit += cw[2] * z3;
if (nFeat >= 4) logit += cw[3] * z4;
+ if (nFeat >= 5) logit += cw[4] * z5;
+ if (nFeat >= 6) logit += cw[5] * z6;
+ if (nFeat >= 7) logit += cw[6] * z7;
+ if (nFeat >= 8) logit += cw[7] * z8;
+ if (nFeat >= 9) logit += cw[8] * z9;
return logit;
}
return (z1 + z2 + z3 + z4) / 4.0f; // fallback: equal weight
@@ -721,14 +863,14 @@ public final class JunkDetector implements TextQualityDetector {
/**
* Mean log-prob over the codepoint pairs in {@code text} using the given
- * script's V7 F1 tables.
+ * script's bigram tables.
*
* <p>For each adjacent codepoint pair {@code (a, b)}:
* <ol>
* <li>Binary-search both codepoints in the script's codepoint index.
* If either is absent, the pair was never seen in training; emit
* {@code α * (logP(a) + logP(b))} using each codepoint's unigram
- * value (or {@link V7Tables#unigramFallbackLogProb} if the
+ * value (or {@link BigramTables#unigramFallbackLogProb} if the
* codepoint isn't even in the unigram index).</li>
* <li>Otherwise, look up the packed {@code (idxA<<16)|idxB} key in
* the open-addressing bigram table. Empty slot → unseen pair →
@@ -736,7 +878,7 @@ public final class JunkDetector implements TextQualityDetector {
* value.</li>
* </ol>
*
- * <p>This is the single authoritative implementation of the V7 F1
+ * <p>This is the single authoritative implementation of the bigram
* scoring math, shared by inference and training. Keeping one
* implementation eliminates the risk of train/infer drift in the F1
* feature.
@@ -744,20 +886,39 @@ public final class JunkDetector implements TextQualityDetector {
* @return mean log-prob, or {@link Double#NaN} if {@code text} has fewer
* than two codepoints or {@code tables} is null
*/
- public static double computeF1MeanLogP(String text, V7Tables tables) {
+ public static double computeF1MeanLogP(String text, BigramTables tables) {
if (text == null || text.length() < 2 || tables == null) {
return Double.NaN;
}
+ return computeF1MeanLogP(text.codePoints().toArray(), tables);
+ }
+
+ /**
+ * Codepoint-array form of {@link #computeF1MeanLogP(String, BigramTables)}.
+ *
+ * <p>Operates on a pre-decoded codepoint sequence rather than a
+ * {@code String} so callers (e.g. {@link #forEachScriptBigram} bucketing)
+ * can score arbitrary codepoint pairs without re-encoding. Same math —
+ * this is the single authoritative implementation; the {@code String}
+ * overload just decodes and delegates.
+ */
+ public static double computeF1MeanLogP(int[] cps, BigramTables tables) {
+ if (cps == null || cps.length < 2 || tables == null) {
+ return Double.NaN;
+ }
+ // Every adjacent pair is scored. The old whitespace+whitespace skip
+ // (HTML-indentation guard) is gone: whitespace is now COMMON and lives
+ // in its own COMMON run/table, so it no longer pollutes a per-script
+ // mean. Dropping the skip also makes the training tally trivially
+ // match this scorer — both just count every adjacent pair.
double sum = 0;
int n = 0;
int prevCp = -1;
int prevIdx = -1;
- for (int i = 0; i < text.length(); ) {
- int cp = text.codePointAt(i);
- i += Character.charCount(cp);
+ for (int cp : cps) {
int curIdx = codepointToIndex(tables, cp);
if (prevCp >= 0) {
- sum += scorePairF1V7(prevCp, prevIdx, cp, curIdx, tables);
+ sum += scorePairF1(prevCp, prevIdx, cp, curIdx, tables);
n++;
}
prevCp = cp;
@@ -772,7 +933,7 @@ public final class JunkDetector implements TextQualityDetector {
* @return the dense index (≥ 0) if found, or -1 if the codepoint
* doesn't appear in any kept bigram for this script
*/
- public static int codepointToIndex(V7Tables tables, int cp) {
+ public static int codepointToIndex(BigramTables tables, int cp) {
return java.util.Arrays.binarySearch(tables.codepointIndex, cp);
}
@@ -812,8 +973,8 @@ public final class JunkDetector implements TextQualityDetector {
* once when scanning the text (avoiding a redundant binary search per
* codepoint).
*/
- private static double scorePairF1V7(int cpA, int idxA, int cpB, int idxB,
- V7Tables tables) {
+ private static double scorePairF1(int cpA, int idxA, int cpB, int idxB,
+ BigramTables tables) {
if (idxA >= 0 && idxB >= 0) {
int slot = lookupBigramSlot(tables, idxA, idxB);
if (slot >= 0) {
@@ -822,7 +983,7 @@ public final class JunkDetector implements TextQualityDetector {
}
}
// Unigram backoff for unseen pair or for codepoints absent from the
- // per-script index. α=1.0 = plain independence; prototype-validated.
+ // per-script index. α=1.0 = plain independence.
double ua = unigramLogProb(tables, idxA);
double ub = unigramLogProb(tables, idxB);
return tables.backoffAlpha * (ua + ub);
@@ -836,20 +997,20 @@ public final class JunkDetector implements TextQualityDetector {
* <p>Linear probing with the same mix-hash used at training time —
* required for the table to be readable, not just writable.
*/
- static int lookupBigramSlot(V7Tables tables, int idxA, int idxB) {
+ static int lookupBigramSlot(BigramTables tables, int idxA, int idxB) {
int packedKey = packBigramKey(idxA, idxB);
int[] keys = tables.bigramKeys;
int mask = keys.length - 1;
int h = mixIndexKey(packedKey) & mask;
while (true) {
int k = keys[h];
- if (k == V7Tables.EMPTY_KEY) return -1;
+ if (k == BigramTables.EMPTY_KEY) return -1;
if (k == packedKey) return h;
h = (h + 1) & mask;
}
}
- private static double unigramLogProb(V7Tables tables, int idx) {
+ private static double unigramLogProb(BigramTables tables, int idx) {
if (idx < 0) {
return tables.unigramFallbackLogProb;
}
@@ -864,73 +1025,192 @@ public final class JunkDetector implements TextQualityDetector {
/**
* Computes the global script-transition z-score for the whole input
- * string against this model's loaded tables. Thin wrapper around the
- * public static {@link #computeZ4ScriptTransition} helper — same math,
- * just preloaded with this instance's parameters.
+ * string against this model's loaded tables. Uses the int16-quantized
+ * lookup; the public static {@link #computeZ4ScriptTransition}
+ * float[] variant remains for trainer use.
*/
private float computeScriptTransitionZ(String text) {
- return computeZ4ScriptTransition(text,
- scriptTransitionTable, scriptTransitionCalibration,
- scriptBucketIndex, numScriptBuckets);
- }
-
- /**
- * Splits text into maximal runs of the same Unicode script.
- * COMMON, INHERITED, and UNKNOWN codepoints (spaces, punctuation, digits)
- * are attached to the preceding script run so that inter-word bigrams are
- * preserved within each run. Any leading COMMON characters are prepended
- * to the first non-COMMON run.
- */
- private List<ScriptRun> buildScriptRuns(String text) {
- List<ScriptRun> runs = new ArrayList<>();
- String currentScript = null;
- StringBuilder currentText = new StringBuilder();
- StringBuilder leadingCommon = new StringBuilder();
-
+ if (scriptTransitionTable == null || scriptTransitionCalibration == null
+ || scriptBucketIndex == null || numScriptBuckets == 0) {
+ return 0f;
+ }
+ int otherBucket = numScriptBuckets - 1;
+ float min = scriptTransitionTableQuant[0];
+ float scale = (scriptTransitionTableQuant[1] - min) / 65535f;
+ int prev = -1;
+ double sum = 0;
+ int count = 0;
for (int i = 0; i < text.length(); ) {
int cp = text.codePointAt(i);
i += Character.charCount(cp);
-
Character.UnicodeScript s = Character.UnicodeScript.of(cp);
if (s == Character.UnicodeScript.COMMON
|| s == Character.UnicodeScript.INHERITED
|| s == Character.UnicodeScript.UNKNOWN) {
- if (currentScript != null) {
- currentText.appendCodePoint(cp);
- } else {
- leadingCommon.appendCodePoint(cp);
- }
continue;
}
-
- String scriptName = SCRIPT_MODEL_FALLBACK.getOrDefault(s.name(), s.name());
-
- if (!scriptName.equals(currentScript)) {
- if (currentScript != null && currentText.length() > 0) {
- runs.add(new ScriptRun(currentScript, currentText.toString()));
- }
- currentScript = scriptName;
- currentText = new StringBuilder();
- if (leadingCommon.length() > 0) {
- currentText.append(leadingCommon);
- leadingCommon.setLength(0);
- }
+ int bucket = scriptBucketIndex.getOrDefault(s.name(), otherBucket);
+ if (prev >= 0) {
+ int val = scriptTransitionTable[prev * numScriptBuckets + bucket] & 0xFFFF;
+ sum += min + val * scale;
+ count++;
}
- currentText.appendCodePoint(cp);
+ prev = bucket;
+ }
+ if (count == 0) {
+ return 0f;
+ }
+ return ((float) (sum / count) - scriptTransitionCalibration[0])
+ / scriptTransitionCalibration[1];
+ }
+
+
+ /** Model script key for the pooled COMMON (digits/punctuation/symbols) table. */
+ public static final String COMMON_SCRIPT = "COMMON";
+
+ /** COMMON-class predicate: COMMON, INHERITED, UNKNOWN all pool into COMMON. */
+ static String classKey(int cp) {
+ Character.UnicodeScript s = Character.UnicodeScript.of(cp);
+ if (s == Character.UnicodeScript.COMMON
+ || s == Character.UnicodeScript.INHERITED
+ || s == Character.UnicodeScript.UNKNOWN) {
+ return COMMON_SCRIPT;
}
+ return SCRIPT_MODEL_FALLBACK.getOrDefault(s.name(), s.name());
+ }
+
+ /** Per-script F1 bigram tables (package-private, for diagnostics). */
+ BigramTables f1TablesFor(String script) {
+ return f1TablesByScript.get(script);
+ }
+
+ /** Per-script z1 calibration {mu, sigma} (package-private, for diagnostics). */
+ float[] calibrationFor(String script) {
+ return calibrations.get(script);
+ }
- if (currentScript != null && currentText.length() > 0) {
- runs.add(new ScriptRun(currentScript, currentText.toString()));
+ /** Per-script F1 bigram tables view (package-private, for diagnostics). */
+ Map<String, BigramTables> f1TablesByScriptView() {
+ return f1TablesByScript;
+ }
+
+ // -----------------------------------------------------------------------
+ // Bucket-by-script bigram enumeration (the keystone).
+ // Single source of truth for BOTH inference z1 scoring and training tally.
+ // A bigram (a,b) is:
+ // - skipped if either codepoint is charset-invariant (digit) — digits
+ // never enter any bucket;
+ // - assigned to the adjacent real script when one side is COMMON glue
+ // (space/punct fold into the neighbouring word);
+ // - assigned to the COMMON bucket when BOTH sides are COMMON — so that
+ // symbol/punctuation salad ("*^&(...") scores against the COMMON table
+ // (unseen → junky), while a formatted number (digits skipped, its
+ // punctuation digit-adjacent) lands almost nothing → neutral;
+ // - skipped if a and b are two DIFFERENT real scripts (cross-script
+ // boundary — not comparable, would mix tables).
+ // COMMON is a normal bucket (its own table + calibration + combiner weight),
+ // scored through this same path.
+ // -----------------------------------------------------------------------
+
+ /** Sink for {@link #forEachScriptBigram}: (modelScript, cpA, cpB). */
+ @FunctionalInterface
+ public interface BigramSink {
+ void accept(String script, int a, int b);
+ }
+
+ /** Charset-invariant content excluded from per-script bigram scoring. */
+ static boolean isSkipCodepoint(int cp) {
+ return Character.isDigit(cp);
+ }
+
+ /**
+ * Enumerates the script-bucketed bigrams of {@code cps} under the redesign
+ * representation (see block comment above). Used identically by inference
+ * (score each emitted bigram) and training (tally each emitted bigram), so
+ * the two can never drift.
+ */
+ /**
+ * Per-script {@code {sumLogP, count}} buckets for a codepoint sequence,
+ * scored with the given per-script tables. This is exactly the z1 input
+ * {@link #aggregate} computes, exposed so the trainer's calibration and the
+ * inference scorer cannot drift.
+ */
+ public static Map<String, double[]> bucketSumsAndCounts(
+ int[] cps, Map<String, BigramTables> tablesByScript) {
+ Map<String, double[]> buckets = new HashMap<>();
+ forEachScriptBigram(cps, (script, a, b) -> {
+ BigramTables t = tablesByScript.get(script);
+ if (t == null) {
+ return;
+ }
+ double lp = computeF1MeanLogP(new int[]{a, b}, t);
+ if (Double.isNaN(lp)) {
+ return;
+ }
+ double[] bk = buckets.computeIfAbsent(script, k -> new double[2]);
+ bk[0] += lp;
+ bk[1] += 1;
+ });
+ return buckets;
+ }
+
+ /** "Real" structural whitespace collapses to canonical U+0020 before bigram
+ * emission. Matches {@link #computeZ3ControlByte}'s definition of
+ * non-anomalous whitespace: HT (0x09), LF (0x0A), CR (0x0D), regular
+ * space (0x20), plus the Zs/Zl/Zp Unicode categories (NBSP, ideographic
+ * space, line/paragraph separators).
+ *
+ * <p><strong>Anomalous Cc (0x01-0x08, 0x0B, 0x0C, 0x0E-0x1F, 0x7F, U+0085
+ * NEL, U+0080-0x009F C1 controls) and Cf (format chars) are DELIBERATELY
+ * NOT normalized.</strong> Their OOV-floor signal is carrying real
+ * evidence that the decode is wrong — e.g., windows-1252 bytes 0x80-0x9F
+ * decode to printable curly quotes / em-dashes; ISO-8859-16 misdecodes
+ * them as C1 control codepoints; the bigram-table OOV-floor on those
+ * Cc-touching bigrams is what correctly penalizes the wrong decode.
+ * z3 has had this distinction since v15; this brings z1 in line. */
+ static int normalizeWhitespace(int cp) {
+ if (cp == 0x20) {
+ return cp;
+ }
+ if (cp == 0x09 || cp == 0x0A || cp == 0x0D) {
+ return 0x20;
+ }
+ int t = Character.getType(cp);
+ if (t == Character.SPACE_SEPARATOR
+ || t == Character.LINE_SEPARATOR
+ || t == Character.PARAGRAPH_SEPARATOR) {
+ return 0x20;
}
- return runs;
+ return cp;
}
- private static final class ScriptRun {
- final String script;
- final String text;
- ScriptRun(String script, String text) {
- this.script = script;
- this.text = text;
+ public static void forEachScriptBigram(int[] cps, BigramSink sink) {
+ if (cps == null || cps.length < 2) {
+ return;
+ }
+ for (int i = 0; i + 1 < cps.length; i++) {
+ int a = normalizeWhitespace(cps[i]);
+ int b = normalizeWhitespace(cps[i + 1]);
+ if (isSkipCodepoint(a) || isSkipCodepoint(b)) {
+ continue;
+ }
+ String ka = classKey(a);
+ String kb = classKey(b);
+ boolean aCommon = COMMON_SCRIPT.equals(ka);
+ boolean bCommon = COMMON_SCRIPT.equals(kb);
+ String script;
+ if (aCommon && bCommon) {
+ script = COMMON_SCRIPT; // symbol/punct salad → COMMON bucket
+ } else if (aCommon) {
+ script = kb; // glue folds into the real side
+ } else if (bCommon) {
+ script = ka;
+ } else if (ka.equals(kb)) {
+ script = ka; // same real script
+ } else {
+ continue; // cross-script boundary
+ }
+ sink.accept(script, a, b);
}
}
diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java
index 72e51e8094..056c768a65 100644
--- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java
+++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java
@@ -20,13 +20,10 @@ import java.io.IOException;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.Collections;
-import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,9 +34,8 @@ import org.apache.tika.detect.EncodingResult;
import org.apache.tika.detect.MetaEncodingDetector;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
-import org.apache.tika.ml.chardetect.HtmlByteStripper;
+import org.apache.tika.ml.chardetect.AdaptiveProbe;
import org.apache.tika.parser.ParseContext;
-import org.apache.tika.quality.TextQualityComparison;
import org.apache.tika.quality.TextQualityDetector;
/**
@@ -51,11 +47,11 @@ import org.apache.tika.quality.TextQualityDetector;
* {@link org.apache.tika.detect.CompositeEncodingDetector} chain emits
* candidates into the {@link EncodingDetectorContext}. This meta detector
* then reads the raw probe bytes, decodes them under each unique candidate
- * charset, and runs pairwise comparisons via
- * {@link TextQualityDetector#compare} to pick the candidate whose decoding
- * produces the cleanest text. BOM-declared, meta-tag-declared,
- * structural, and statistical candidates all compete on the same footing —
- * quality of the resulting decode is the sole criterion at this layer.
+ * charset, scores each decode with {@link TextQualityDetector#score}, and
+ * picks the highest-scoring candidate (the score is cross-script comparable,
+ * so a plain argmax suffices). BOM-declared, meta-tag-declared, structural,
+ * and statistical candidates all compete on the same footing — quality of the
+ * resulting decode is the sole criterion at this layer.
*
* <p>The {@link TextQualityDetector} implementation is discovered via
* {@link ServiceLoader}. When no implementation is on the classpath,
@@ -77,6 +73,13 @@ public class JunkFilterEncodingDetector implements MetaEncodingDetector {
* default read limit used by the charset base detectors. */
private static final int DEFAULT_READ_LIMIT = 16384;
+ /** A STATISTICAL candidate at or below this confidence carries no real
+ * signal — it's the "I don't know" level (matches Mojibuster's
+ * windows-1252 fallback confidence). When the statistical layer offers
+ * nothing above this, the junk-filter defers to a DECLARATIVE/STRUCTURAL
+ * anchor instead of arbitrating near-identical decodes by quality. */
+ private static final float NO_INFO_CONFIDENCE = 0.1f;
+
/** Cached quality detector. {@code null} if none is on the classpath. */
private final TextQualityDetector qualityDetector;
@@ -143,38 +146,18 @@ public class JunkFilterEncodingDetector implements MetaEncodingDetector {
}
bytes = stripBomBytes(bytes);
- // Strip HTML/XML markup before decoding so the quality score reflects
- // body text, not whitespace and tags. Falls back to the raw probe
- // when no well-formed tags are detected.
- byte[] forDecode = bytes;
- byte[] stripDst = new byte[bytes.length];
- HtmlByteStripper.Result stripped =
- HtmlByteStripper.strip(bytes, 0, bytes.length, stripDst, 0);
- boolean stripUsed = stripped.tagCount > 0 && stripped.length > 0;
- LOG.trace("junk-filter strip: input={}B tagCount={} stripped={}B used={}",
- bytes.length, stripped.tagCount, stripped.length, stripUsed);
- if (stripUsed) {
- forDecode = new byte[stripped.length];
- System.arraycopy(stripDst, 0, forDecode, 0, stripped.length);
- }
-
- // Decode probe under each candidate, preserving insertion order so
- // tournament seeding is deterministic.
- //
- // Each decoded string is then run through HTML entity expansion.
- // For entity-encoded HTML (numeric refs like ്), this is
- // load-bearing: entity refs are ASCII bytes that decode identically
- // under every candidate charset, so they don't differentiate.
- // After expansion they become real codepoints — and crucially, in
- // the *wrong* decoding (e.g. mojibake-as-HAN), they introduce
- // cross-script transitions (HAN ↔ MALAYALAM mid-document) that the
- // quality detector's script-transition feature correctly penalises.
- // See `20260512-junkdetector-codepoint-hash-plan.md` (AIT5 case).
+ // Decode each candidate, then HtmlContentCleaner.clean — the same
+ // tag-strip + entity-expand TrainJunkModel applies, so train and
+ // inference match. Entity expansion is load-bearing: numeric refs
+ // become codepoints whose cross-script transitions expose mojibake
+ // under a wrong decoding (AIT5 case).
Map<Charset, String> candidates = new LinkedHashMap<>();
for (Charset cs : uniqueCharsets) {
- String decoded = safeDecode(forDecode, cs);
+ String decoded = safeDecode(bytes, cs);
+ if (decoded != null && !decoded.isEmpty()) {
+ decoded = HtmlContentCleaner.clean(decoded);
+ }
if (decoded != null && !decoded.isEmpty()) {
- decoded = expandHtmlEntities(decoded);
candidates.put(cs, decoded);
if (LOG.isTraceEnabled()) {
int sampleLen = Math.min(400, decoded.length());
@@ -216,29 +199,129 @@ public class JunkFilterEncodingDetector implements MetaEncodingDetector {
return Collections.emptyList();
}
- // Pairwise tournament: the first candidate seeds the champion slot;
- // every subsequent candidate challenges the current champion.
- Iterator<Map.Entry<Charset, String>> it = candidates.entrySet().iterator();
- Map.Entry<Charset, String> champion = it.next();
- LOG.trace("junk-filter tournament seed: {}", champion.getKey().name());
- while (it.hasNext()) {
- Map.Entry<Charset, String> challenger = it.next();
- TextQualityComparison cmp = qualityDetector.compare(
- champion.getKey().name(), champion.getValue(),
- challenger.getKey().name(), challenger.getValue());
- LOG.trace("junk-filter compare {} vs {} -> {} (delta={} A={} B={})",
- champion.getKey().name(), challenger.getKey().name(),
- cmp.winner(), String.format(java.util.Locale.ROOT, "%.3f", cmp.delta()),
- cmp.scoreA(), cmp.scoreB());
- if (challenger.getKey().name().equals(cmp.winner())) {
- champion = challenger;
+ // Calibrated-rescale argmax. Score each candidate once with the
+ // quality detector, rescale per-script to a [junk≈0, clean≈1]
+ // common scale, then pick the highest. The rescaling is what
+ // makes cross-script comparisons sound — without it, the more
+ // permissive HAN/LATIN classifiers can out-score the stricter
+ // HANGUL/ARABIC/HEBREW ones on equal-quality text and arbitrate
+ // wrong (the Korean→Chinese case).
+ //
+ // Operates on raw decoded candidates — the strip-COMMON step that
+ // used to live here was removed once γ (whitespace-bigram skip)
+ // and NFC normalization landed inside JunkDetector itself. Those
+ // address the same Masada-style whitespace-storm root cause for
+ // every caller of JunkDetector and avoid the train/inference
+ // distribution divergence that the strip introduced.
+ // The JunkDetector logit is cross-script comparable (z1 calibrated per
+ // script, z2..z9 global), so the base decision is a plain argmax of the
+ // raw score. BUT the score is an ABSOLUTE per-decode quality, dominated
+ // by shared content (whitespace/digits identical across decodes); on a
+ // COMMON-dominated doc the discriminating bytes are diluted and the top
+ // candidates differ only by noise. The quality signal is STATISTICAL-
+ // grade evidence, so it may override a higher-evidence anchor
+ // (DECLARATIVE author intent, or STRUCTURAL byte-grammar proof) only when
+ // it beats that anchor's score by OVERRIDE_MARGIN; otherwise we defer to
+ // the anchor. This is honest low-confidence behaviour, not a tie-break:
+ // where the model has real signal (e.g. UTF-8 over garbage UTF-16, Δ≫1)
+ // it still overrides freely.
+ Charset champion = null;
+ double championZ = Double.NEGATIVE_INFINITY;
+ Map<Charset, Double> scoreByCharset = new LinkedHashMap<>();
+ for (Map.Entry<Charset, String> entry : candidates.entrySet()) {
+ org.apache.tika.quality.TextQualityScore sc =
+ qualityDetector.score(entry.getValue());
+ float rawZ = sc.isUnknown() ? Float.NEGATIVE_INFINITY : sc.getZScore();
+ scoreByCharset.put(entry.getKey(), (double) rawZ);
+ LOG.trace("junk-filter score {} z={} script={}",
+ entry.getKey().name(),
+ String.format(java.util.Locale.ROOT, "%.3f", rawZ),
+ sc.isUnknown() ? "UNKNOWN" : sc.getDominantScript());
+ if (rawZ > championZ) {
+ championZ = rawZ;
+ champion = entry.getKey();
}
}
- LOG.trace("junk-filter -> {} (tournament champion)", champion.getKey().name());
+ if (champion == null) {
+ // Every candidate scored UNKNOWN (no modelable script) — the filter
+ // has no opinion, so keep the first (highest-confidence) candidate.
+ champion = candidates.keySet().iterator().next();
+ }
- float confidence = context.getTopConfidenceFor(champion.getKey());
+ // "No-info" guard: if the statistical layer produced no confident
+ // answer — no STRUCTURAL proof, and its best STATISTICAL candidate is
+ // no better than Mojibuster's windows-1252 "I don't know" fallback
+ // (confidence <= NO_INFO_CONFIDENCE) — then it has nothing to say, so a
+ // DECLARATIVE/STRUCTURAL anchor (the author's declaration) should win
+ // rather than a quality argmax over near-identical decodes. Fires ONLY
+ // when the statistical layer abstained, so it cannot cost the
+ // confident-detection wins (UTF-8 recovery etc.).
+ Charset anchor = bestAnchor(context, scoreByCharset);
+ if (anchor != null && !anchor.equals(champion)
+ && !hasConfidentNonDeclarative(context)) {
+ LOG.trace("junk-filter -> {} (defer to anchor; statistical layer gave "
+ + "no confident answer, champion was {})",
+ anchor.name(), champion.name());
+ context.setArbitrationInfo("junk-filter-defer-no-info");
+ return List.of(new EncodingResult(anchor, context.getTopConfidenceFor(anchor)));
+ }
+ LOG.trace("junk-filter -> {} (argmax z={})",
+ champion.name(),
+ String.format(java.util.Locale.ROOT, "%.3f", championZ));
+
+ float confidence = context.getTopConfidenceFor(champion);
context.setArbitrationInfo("junk-filter-selected");
- return List.of(new EncodingResult(champion.getKey(), confidence));
+ return List.of(new EncodingResult(champion, confidence));
+ }
+
+ /**
+ * True if some detector produced a confident non-declarative signal: any
+ * STRUCTURAL result (byte-grammar proof), or any STATISTICAL result above
+ * {@link #NO_INFO_CONFIDENCE}. When false, the statistical layer has
+ * effectively abstained (only its "I don't know" fallback), so a
+ * declaration should be trusted over a quality argmax.
+ */
+ private static boolean hasConfidentNonDeclarative(EncodingDetectorContext context) {
+ for (EncodingDetectorContext.Result r : context.getResults()) {
+ for (EncodingResult er : r.getEncodingResults()) {
+ EncodingResult.ResultType t = er.getResultType();
+ if (t == EncodingResult.ResultType.STRUCTURAL) {
+ return true;
+ }
+ if (t == EncodingResult.ResultType.STATISTICAL
+ && er.getConfidence() > NO_INFO_CONFIDENCE) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Highest-scoring DECLARATIVE/STRUCTURAL candidate present in the scored
+ * pool, or {@code null} if none. This is the higher-evidence "anchor" the
+ * junk-filter defers to when the statistical layer gives no confident answer
+ * (see {@link #hasConfidentNonDeclarative}).
+ */
+ private static Charset bestAnchor(EncodingDetectorContext context,
+ Map<Charset, Double> scoreByCharset) {
+ Charset best = null;
+ double bestZ = Double.NEGATIVE_INFINITY;
+ for (EncodingDetectorContext.Result r : context.getResults()) {
+ for (EncodingResult er : r.getEncodingResults()) {
+ EncodingResult.ResultType t = er.getResultType();
+ if (t != EncodingResult.ResultType.DECLARATIVE
+ && t != EncodingResult.ResultType.STRUCTURAL) {
+ continue;
+ }
+ Double z = scoreByCharset.get(er.getCharset());
+ if (z != null && z > bestZ) {
+ bestZ = z;
+ best = er.getCharset();
+ }
+ }
+ }
+ return best;
}
/**
@@ -280,27 +363,9 @@ public class JunkFilterEncodingDetector implements MetaEncodingDetector {
}
private byte[] readProbe(TikaInputStream tis) throws IOException {
- try {
- tis.mark(readLimit);
- byte[] buf = new byte[readLimit];
- int total = 0;
- int read;
- while (total < readLimit
- && (read = tis.read(buf, total, readLimit - total)) != -1) {
- total += read;
- }
- if (total == 0) {
- return null;
- }
- if (total < readLimit) {
- byte[] trimmed = new byte[total];
- System.arraycopy(buf, 0, trimmed, 0, total);
- return trimmed;
- }
- return buf;
- } finally {
- tis.reset();
- }
+ // readLimit is the tag-stripped content target; cap raw reads at 512 KB.
+ byte[] probe = AdaptiveProbe.read(tis, readLimit, AdaptiveProbe.DEFAULT_RAW_CAP);
+ return probe.length == 0 ? null : probe;
}
private static String safeDecode(byte[] bytes, Charset charset) {
@@ -323,56 +388,13 @@ public class JunkFilterEncodingDetector implements MetaEncodingDetector {
// intermixed with raw UTF-8 codepoints.
// -----------------------------------------------------------------------
- private static final Pattern ENTITY_DEC = Pattern.compile("&#(\\d{1,7});");
- private static final Pattern ENTITY_HEX = Pattern.compile("&#[xX]([0-9a-fA-F]{1,6});");
- private static final Pattern ENTITY_NAMED =
- Pattern.compile("&(amp|lt|gt|quot|apos|nbsp|copy|reg);");
-
/**
- * Expands HTML numeric and a small set of named entity references in
- * {@code s}. Malformed or out-of-range entities pass through unchanged.
- * The named-entity set is intentionally small — only the universally-
- * declared HTML5 entities that don't depend on a DOCTYPE. Anything more
- * exotic stays as a literal entity reference (which scores as ASCII noise,
- * the same as it would have before).
+ * Delegates to {@link HtmlContentCleaner#expandHtmlEntities} — the single
+ * implementation shared with training. Retained here as the historical
+ * entry point used by tests and diagnostics.
*/
static String expandHtmlEntities(String s) {
- s = ENTITY_DEC.matcher(s).replaceAll(mr -> {
- try {
- int cp = Integer.parseInt(mr.group(1));
- if (cp >= 0 && cp <= 0x10FFFF) {
- return Matcher.quoteReplacement(new String(Character.toChars(cp)));
- }
- } catch (NumberFormatException ignored) {
- // overflow — fall through, leave entity literal
- }
- return Matcher.quoteReplacement(mr.group());
- });
- s = ENTITY_HEX.matcher(s).replaceAll(mr -> {
- try {
- int cp = Integer.parseInt(mr.group(1), 16);
- if (cp >= 0 && cp <= 0x10FFFF) {
- return Matcher.quoteReplacement(new String(Character.toChars(cp)));
- }
- } catch (NumberFormatException ignored) {
- // overflow — fall through, leave entity literal
- }
- return Matcher.quoteReplacement(mr.group());
- });
- s = ENTITY_NAMED.matcher(s).replaceAll(mr -> {
- switch (mr.group(1)) {
- case "amp": return "&";
- case "lt": return "<";
- case "gt": return ">";
- case "quot": return "\"";
- case "apos": return "'";
- case "nbsp": return " ";
- case "copy": return "©";
- case "reg": return "®";
- default: return Matcher.quoteReplacement(mr.group());
- }
- });
- return s;
+ return HtmlContentCleaner.expandHtmlEntities(s);
}
/**
diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/TextQualityFeatures.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/TextQualityFeatures.java
new file mode 100644
index 0000000000..25f07b765f
--- /dev/null
+++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/TextQualityFeatures.java
@@ -0,0 +1,608 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.junkdetect;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Candidate charset-agnostic features for use in the Phase-2 feature study
+ * (see plan {@code wild-roaming-whale}). All features are pure functions of a
+ * decoded {@code String} so they can be computed by the eval harness without
+ * touching the trained bigram model. Intended to migrate into
+ * {@link JunkDetector}'s feature vector after the study identifies the
+ * non-redundant subset.
+ *
+ * <p>Also hosts the {@link StripMode} enum and {@link #strip} entry point.
+ * The current production behaviour
+ * ({@code JunkFilterEncodingDetector.stripCommonCodepoints}) corresponds to
+ * {@link StripMode#ALL_COMMON} — it strips every COMMON / INHERITED /
+ * UNKNOWN codepoint before scoring. The user observation in the plan was
+ * that this is too aggressive: it removes pilcrows and other punctuation
+ * marks that are themselves mojibake fingerprints. The other modes let the
+ * eval measure that empirically.
+ */
+public final class TextQualityFeatures {
+
+ private TextQualityFeatures() {
+ }
+
+ // -----------------------------------------------------------------------
+ // Strip modes
+ // -----------------------------------------------------------------------
+
+ public enum StripMode {
+ /** No stripping — pass text through unchanged. */
+ NONE,
+ /**
+ * Strip only Unicode whitespace (per
+ * {@link Character#isWhitespace(int)}). Keeps punctuation, digits,
+ * Latin-1 Supplement symbols (¶ © ÷ etc.) — the signals that
+ * distinguish mojibake from clean text.
+ */
+ WHITESPACE,
+ /**
+ * Strip whitespace plus control characters and format characters
+ * (general categories Cc, Cf). Still keeps printable punctuation.
+ */
+ WHITESPACE_CONTROL,
+ /**
+ * Production behaviour today: strip every COMMON, INHERITED, and
+ * UNKNOWN-script codepoint. Removes everything in the BMP that is
+ * not script-tagged — including printable punctuation, digits, and
+ * Latin-1 supplement symbols.
+ */
+ ALL_COMMON
+ }
+
+ public static String strip(String text, StripMode mode) {
+ if (text == null || text.isEmpty() || mode == StripMode.NONE) {
+ return text;
+ }
+ StringBuilder sb = new StringBuilder(text.length());
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ i += Character.charCount(cp);
+ if (shouldStrip(cp, mode)) {
+ continue;
+ }
+ sb.appendCodePoint(cp);
+ }
+ return sb.toString();
+ }
+
+ private static boolean shouldStrip(int cp, StripMode mode) {
+ switch (mode) {
+ case WHITESPACE:
+ return Character.isWhitespace(cp);
+ case WHITESPACE_CONTROL: {
+ if (Character.isWhitespace(cp)) {
+ return true;
+ }
+ int type = Character.getType(cp);
+ return type == Character.CONTROL || type == Character.FORMAT;
+ }
+ case ALL_COMMON: {
+ Character.UnicodeScript s = Character.UnicodeScript.of(cp);
+ return s == Character.UnicodeScript.COMMON
+ || s == Character.UnicodeScript.INHERITED
+ || s == Character.UnicodeScript.UNKNOWN;
+ }
+ default:
+ return false;
+ }
+ }
+
+ // -----------------------------------------------------------------------
+ // Candidate features (z5..z9 of the plan)
+ // -----------------------------------------------------------------------
+
+ /**
+ * z6: fraction of codepoints that are letters
+ * ({@link Character#isLetter(int)}). Polish {@code ciśnienia} ≈ 1.0;
+ * {@code ci¶nienia} < 1.0 because {@code ¶} is not a letter.
+ */
+ public static double alphabeticRatio(String text) {
+ if (text == null || text.isEmpty()) {
+ return Double.NaN;
+ }
+ int total = 0;
+ int letters = 0;
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ i += Character.charCount(cp);
+ total++;
+ if (Character.isLetter(cp)) {
+ letters++;
+ }
+ }
+ return total == 0 ? Double.NaN : (double) letters / total;
+ }
+
+ /**
+ * z5: fraction of adjacent codepoint pairs where both codepoints are
+ * letters in the same script cluster. Script cluster groups
+ * HAN + HIRAGANA + KATAKANA + HANGUL + BOPOMOFO (CJK) into one cluster
+ * so Japanese mixed text and Korean Hanja text count as same-cluster
+ * pairs; all other scripts are their own cluster.
+ *
+ * <p>Polish {@code ciśnienia} → 1.0 (every adjacent pair is two LATIN
+ * letters). {@code ci¶nienia} → 0.75 (the two pairs involving {@code ¶}
+ * fail the both-letters test).
+ */
+ public static double letterPairDensity(String text) {
+ if (text == null || text.length() < 2) {
+ return Double.NaN;
+ }
+ int[] cps = text.codePoints().toArray();
+ if (cps.length < 2) {
+ return Double.NaN;
+ }
+ int pairs = 0;
+ int matches = 0;
+ for (int i = 0; i + 1 < cps.length; i++) {
+ int a = cps[i];
+ int b = cps[i + 1];
+ pairs++;
+ if (Character.isLetter(a) && Character.isLetter(b)
+ && sameScriptCluster(a, b)) {
+ matches++;
+ }
+ }
+ return pairs == 0 ? Double.NaN : (double) matches / pairs;
+ }
+
+ /**
+ * z7: Shannon entropy (in bits) of the distribution of distinct
+ * codepoints in the high-byte range (cp >= 0x80). Clean text uses a
+ * small alphabet there; CJK-as-Latin mojibake (the {@code ƅ} storm)
+ * fans out across many distinct codepoints, raising entropy. Returns
+ * 0 if no high-byte codepoints are present.
+ */
+ public static double highByteEntropy(String text) {
+ if (text == null || text.isEmpty()) {
+ return Double.NaN;
+ }
+ Map<Integer, Integer> counts = new HashMap<>();
+ int total = 0;
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ i += Character.charCount(cp);
+ if (cp >= 0x80) {
+ counts.merge(cp, 1, Integer::sum);
+ total++;
+ }
+ }
+ if (total == 0) {
+ return 0.0;
+ }
+ double h = 0;
+ for (int c : counts.values()) {
+ double p = (double) c / total;
+ h -= p * (Math.log(p) / Math.log(2));
+ }
+ return h;
+ }
+
+ /**
+ * z6 raw input: fraction of codepoints that are "anomaly indicators" —
+ * codepoints that shouldn't appear in correctly-decoded natural text.
+ * Direct decode-failure signal — wrong encodings produce these in bulk
+ * (Java's CharsetDecoder emits U+FFFD per malformed byte; ISO-8859-X
+ * misreads windows-1252 high bytes as C1 controls; PDF cmap failures
+ * emit private-use codepoints; etc.).
+ *
+ * <p>Anomaly set:
+ * <ul>
+ * <li>U+FFFD (REPLACEMENT CHARACTER) — the direct decode-failure marker
+ * <li>Anomalous Cc: {@code 0x01-0x08, 0x0B, 0x0C, 0x0E-0x1F, 0x7F}
+ * (matching z3's byte-level anomaly definition, at codepoint level)
+ * <li>C1 control codepoints: {@code U+0080-U+009F} — the
+ * ISO-8859-X-misdecodes-windows-1252 signal
+ * <li>Private use area: {@code U+E000-U+F8FF}, plus planes 15-16 PUA —
+ * the PDF cmap-failure signal
+ * </ul>
+ *
+ * <p>Continuous (not a binary threshold) so the JunkDetector combiner LR
+ * can learn a proportional weight on it. Excluded: 0x00 (NUL — can
+ * occur legitimately in some text streams; matches z3's exclusion);
+ * 0x09/0x0A/0x0D/0x20 (legitimate whitespace); Cf format chars (ZWJ etc.
+ * have legitimate linguistic uses); Cn unassigned (rare in practice).
+ */
+ public static double replacementRatio(String text) {
+ if (text == null || text.isEmpty()) {
+ return Double.NaN;
+ }
+ int total = 0;
+ int anomaly = 0;
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ i += Character.charCount(cp);
+ total++;
+ if (isAnomalyCodepoint(cp)) {
+ anomaly++;
+ }
+ }
+ return total == 0 ? Double.NaN : (double) anomaly / total;
+ }
+
+ /** True if {@code cp} is in the z6 anomaly set: U+FFFD, anomalous Cc
+ * (matching z3 byte-level definition), C1 controls, or private use. */
+ static boolean isAnomalyCodepoint(int cp) {
+ if (cp == 0xFFFD) {
+ return true;
+ }
+ // Anomalous Cc (excludes 0x00, 0x09, 0x0A, 0x0D — match z3)
+ if ((cp >= 0x01 && cp <= 0x08)
+ || cp == 0x0B || cp == 0x0C
+ || (cp >= 0x0E && cp <= 0x1F)
+ || cp == 0x7F) {
+ return true;
+ }
+ // C1 controls — the ISO-8859-X-misreads-windows-1252 signal
+ if (cp >= 0x0080 && cp <= 0x009F) {
+ return true;
+ }
+ // Private use area (BMP)
+ if (cp >= 0xE000 && cp <= 0xF8FF) {
+ return true;
+ }
+ // Supplementary PUA (planes 15 and 16)
+ if (cp >= 0xF0000 && cp <= 0x10FFFD) {
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Raw count of U+FFFD codepoints. Kept for diagnostics — the per-record
+ * eval TSV emits both the count (for easy spot-checking of "how bad was
+ * this decode") and the ratio (the trainable feature).
+ */
+ public static int replacementCount(String text) {
+ if (text == null || text.isEmpty()) {
+ return 0;
+ }
+ int n = 0;
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ i += Character.charCount(cp);
+ if (cp == 0xFFFD) {
+ n++;
+ }
+ }
+ return n;
+ }
+
+ /**
+ * z10: fraction of codepoints that are combining or spacing marks
+ * (Unicode general categories Mn, Mc, Me). Real Vietnamese / Indic /
+ * Thai / Arabic text uses combining marks heavily (Vietnamese ~30 %);
+ * mojibake from re-decoding precomposed scripts as Latin-1 has zero.
+ * Companion / corrective signal to {@link #alphabeticRatio}, which is
+ * backwards on Vietnamese cohorts because marks aren't letters.
+ * Works on a single 5-codepoint word.
+ */
+ public static double combiningMarkRatio(String text) {
+ if (text == null || text.isEmpty()) {
+ return Double.NaN;
+ }
+ int total = 0;
+ int marks = 0;
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ i += Character.charCount(cp);
+ total++;
+ int type = Character.getType(cp);
+ if (type == Character.NON_SPACING_MARK
+ || type == Character.COMBINING_SPACING_MARK
+ || type == Character.ENCLOSING_MARK) {
+ marks++;
+ }
+ }
+ return total == 0 ? Double.NaN : (double) marks / total;
+ }
+
+ /**
+ * z11: fraction of adjacent codepoint pairs where the first is a
+ * letter and the second is a combining or spacing mark. Bigram-shaped
+ * companion to {@link #combiningMarkRatio} — direct positive signal
+ * for "letter wearing decoration," which is what correct Vietnamese /
+ * Indic / Thai decoding produces and Latin-1 mojibake of those does
+ * not.
+ */
+ public static double letterAdjacentToMarkRatio(String text) {
+ if (text == null || text.length() < 2) {
+ return Double.NaN;
+ }
+ int[] cps = text.codePoints().toArray();
+ if (cps.length < 2) {
+ return Double.NaN;
+ }
+ int pairs = 0;
+ int hits = 0;
+ for (int i = 0; i + 1 < cps.length; i++) {
+ pairs++;
+ int a = cps[i];
+ int b = cps[i + 1];
+ if (Character.isLetter(a)) {
+ int type = Character.getType(b);
+ if (type == Character.NON_SPACING_MARK
+ || type == Character.COMBINING_SPACING_MARK
+ || type == Character.ENCLOSING_MARK) {
+ hits++;
+ }
+ }
+ }
+ return pairs == 0 ? Double.NaN : (double) hits / pairs;
+ }
+
+ /**
+ * Fraction of codepoints assigned to a "real" script (i.e. not in
+ * COMMON / INHERITED / UNKNOWN). Pure-whitespace, pure-digit, and
+ * pure-punctuation text score 0; mostly-letter text scores near 1.
+ *
+ * <p>Used by JunkDetector's "no scoreable script" fallback classifier
+ * (the "NONE" model) to distinguish "real text in an unmodeled
+ * script" (high density, low fragmentation → modestly positive
+ * signal) from "all-whitespace / digit-only content" (zero density
+ * → strong negative signal in JunkDetector's bigram-based judgment,
+ * mild signal for general-purpose junk filtering).
+ */
+ public static double scriptDensity(String text) {
+ if (text == null || text.isEmpty()) {
+ return Double.NaN;
+ }
+ int total = 0;
+ int scripted = 0;
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ i += Character.charCount(cp);
+ total++;
+ Character.UnicodeScript s = Character.UnicodeScript.of(cp);
+ if (s != Character.UnicodeScript.COMMON
+ && s != Character.UnicodeScript.INHERITED
+ && s != Character.UnicodeScript.UNKNOWN) {
+ scripted++;
+ }
+ }
+ return total == 0 ? Double.NaN : (double) scripted / total;
+ }
+
+ /**
+ * Fragmentation of script-bearing codepoints across distinct scripts:
+ * {@code 1 - longest_same_script_run_length / total_script_codepoints}.
+ * Coherent one-script text scores 0 (no fragmentation); script-salad
+ * mojibake (many tiny runs across multiple scripts) approaches 1.
+ *
+ * <p>Combined with {@link #scriptDensity}, distinguishes the four
+ * "no-scoreable-script" failure modes:
+ * <ul>
+ * <li>All-whitespace / pure-digit: density 0, fragmentation 0
+ * (no scripted codepoints at all).</li>
+ * <li>Real Gothic / unmodeled-but-coherent script: density 1,
+ * fragmentation 0 (one long run).</li>
+ * <li>Script-salad mojibake: density > 0.5, fragmentation
+ * > 0.7 (many short runs across many scripts).</li>
+ * <li>Real multilingual text (e.g. Japanese with romaji): density
+ * 1, fragmentation 0.3-0.5 (a handful of long runs).</li>
+ * </ul>
+ *
+ * <p>Returns 0 when text has no script-bearing codepoints (so the
+ * caller can rely on {@link #scriptDensity} to discriminate the
+ * "no-content" case separately).
+ */
+ public static double scriptFragmentation(String text) {
+ if (text == null || text.isEmpty()) {
+ return Double.NaN;
+ }
+ int totalScripted = 0;
+ int longestRun = 0;
+ int currentRun = 0;
+ String currentScript = null;
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ i += Character.charCount(cp);
+ Character.UnicodeScript s = Character.UnicodeScript.of(cp);
+ if (s == Character.UnicodeScript.COMMON
+ || s == Character.UnicodeScript.INHERITED
+ || s == Character.UnicodeScript.UNKNOWN) {
+ continue;
+ }
+ totalScripted++;
+ String name = s.name();
+ if (name.equals(currentScript)) {
+ currentRun++;
+ } else {
+ if (currentRun > longestRun) {
+ longestRun = currentRun;
+ }
+ currentScript = name;
+ currentRun = 1;
+ }
+ }
+ if (currentRun > longestRun) {
+ longestRun = currentRun;
+ }
+ if (totalScripted == 0) {
+ return 0.0; // no scripted content → no fragmentation signal
+ }
+ return 1.0 - (double) longestRun / totalScripted;
+ }
+
+ /**
+ * z9: script-alternation ratio — observed transitions over expected
+ * transitions under a random-shuffle null.
+ *
+ * <p>Formally: for a sequence of n non-COMMON codepoints with script
+ * proportions {@code p_1, ..., p_k}, the expected number of
+ * (transition between different scripts) under random shuffling is
+ * {@code (n - 1) * (1 - sum(p_i^2))} — the second factor is
+ * Gini-Simpson diversity (probability two random positions differ
+ * in script). This is the Wald-Wolfowitz runs-test statistic
+ * generalised to k categories.
+ *
+ * <p>Returns {@code observed_transitions / expected_transitions}:
+ * <ul>
+ * <li>≈ 1 — scripts randomly interleaved (the mojibake signature
+ * when accents are scattered through Latin text — each accent
+ * becomes a singleton Han run, looking random)</li>
+ * <li>< 1 — clumped (normal: words/phrases stay in one script;
+ * English document with embedded Chinese phrase scores 0.05-0.3)</li>
+ * <li>> 1 — more alternating than chance (pathological:
+ * "HLHLHL" patterns)</li>
+ * </ul>
+ *
+ * <p>Length- and proportion-invariant by construction. COMMON /
+ * INHERITED / UNKNOWN codepoints are ignored to keep whitespace
+ * and punctuation from dominating the signal in normal text.
+ *
+ * <p>Returns 0 for single-script documents (no diversity possible).
+ */
+ public static double scriptAlternationRatio(String text) {
+ if (text == null || text.isEmpty()) {
+ return Double.NaN;
+ }
+ // Pass 1: count codepoints per non-COMMON script.
+ java.util.Map<String, Integer> scriptCounts = new java.util.HashMap<>();
+ int totalScripted = 0;
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ i += Character.charCount(cp);
+ Character.UnicodeScript s = Character.UnicodeScript.of(cp);
+ if (s == Character.UnicodeScript.COMMON
+ || s == Character.UnicodeScript.INHERITED
+ || s == Character.UnicodeScript.UNKNOWN) {
+ continue;
+ }
+ scriptCounts.merge(s.name(), 1, Integer::sum);
+ totalScripted++;
+ }
+ if (scriptCounts.size() <= 1 || totalScripted < 2) {
+ return 0.0; // single script (or too short) → no alternation possible
+ }
+
+ // Gini-Simpson diversity = 1 - sum(p_i^2)
+ double sumPiSq = 0;
+ for (int c : scriptCounts.values()) {
+ double p = (double) c / totalScripted;
+ sumPiSq += p * p;
+ }
+ double expectedTransitions = (totalScripted - 1) * (1.0 - sumPiSq);
+ if (expectedTransitions <= 0) {
+ return 0.0;
+ }
+
+ // Pass 2: count observed transitions between distinct non-COMMON scripts.
+ int observedTransitions = 0;
+ String prevScript = null;
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ i += Character.charCount(cp);
+ Character.UnicodeScript s = Character.UnicodeScript.of(cp);
+ if (s == Character.UnicodeScript.COMMON
+ || s == Character.UnicodeScript.INHERITED
+ || s == Character.UnicodeScript.UNKNOWN) {
+ continue;
+ }
+ String name = s.name();
+ if (prevScript != null && !prevScript.equals(name)) {
+ observedTransitions++;
+ }
+ prevScript = name;
+ }
+
+ return observedTransitions / expectedTransitions;
+ }
+
+ /**
+ * Candidate feature (not currently in the classifier): fraction of
+ * whitespace-delimited tokens whose letter codepoints all belong to
+ * the same script cluster. Mojibake often produces tokens with
+ * mixed-script letters (Latin + Cyrillic + Greek in one "word").
+ * Tokens with zero letters are excluded from both numerator and
+ * denominator.
+ */
+ public static double perWordScriptPurity(String text) {
+ if (text == null || text.isEmpty()) {
+ return Double.NaN;
+ }
+ int tokens = 0;
+ int pureTokens = 0;
+ int len = text.length();
+ int i = 0;
+ while (i < len) {
+ // skip whitespace
+ int cp = text.codePointAt(i);
+ if (Character.isWhitespace(cp)) {
+ i += Character.charCount(cp);
+ continue;
+ }
+ // accumulate a token
+ int tokenStart = i;
+ Set<String> clusters = new HashSet<>();
+ int letters = 0;
+ while (i < len) {
+ int c = text.codePointAt(i);
+ if (Character.isWhitespace(c)) {
+ break;
+ }
+ if (Character.isLetter(c)) {
+ letters++;
+ clusters.add(scriptClusterOf(c));
+ }
+ i += Character.charCount(c);
+ }
+ if (letters > 0) {
+ tokens++;
+ if (clusters.size() == 1) {
+ pureTokens++;
+ }
+ }
+ // tokenStart unused; loop continues with i past the token
+ if (i == tokenStart) {
+ // safety: never advance past end without consuming
+ break;
+ }
+ }
+ return tokens == 0 ? Double.NaN : (double) pureTokens / tokens;
+ }
+
+ // -----------------------------------------------------------------------
+ // Script-cluster helper (CJK grouped; others stand alone)
+ // -----------------------------------------------------------------------
+
+ static boolean sameScriptCluster(int cpA, int cpB) {
+ return scriptClusterOf(cpA).equals(scriptClusterOf(cpB));
+ }
+
+ private static String scriptClusterOf(int cp) {
+ Character.UnicodeScript s = Character.UnicodeScript.of(cp);
+ switch (s) {
+ case HAN:
+ case HIRAGANA:
+ case KATAKANA:
+ case HANGUL:
+ case BOPOMOFO:
+ return "CJK";
+ default:
+ return s.name();
+ }
+ }
+}
diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/AnalyzeHanByBlock.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/AnalyzeHanByBlock.java
deleted file mode 100644
index 08b2aa4eb5..0000000000
--- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/AnalyzeHanByBlock.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.ml.junkdetect.tools;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.zip.GZIPInputStream;
-
-/**
- * Diagnostic tool: bucket every bigram in {@code han.train.gz} (or any
- * specified file) by the {@link Character.UnicodeBlock} of each codepoint,
- * and report the distribution.
- *
- * <p>Goal: determine whether HAN's 224K distinct pairs split cleanly along
- * block boundaries — e.g. CJK Unified Ideographs vs. Hiragana vs. Katakana —
- * which would justify routing HAN windows to language-specific sub-models in
- * the v7 design.
- *
- * <p>Usage:
- * <pre>
- * java ... AnalyzeHanByBlock /path/to/junkdetect/han.train.gz
- * </pre>
- */
-public final class AnalyzeHanByBlock {
-
- private AnalyzeHanByBlock() {}
-
- public static void main(String[] args) throws IOException {
- if (args.length < 1) {
- System.err.println("Usage: AnalyzeHanByBlock <train.gz>");
- System.exit(1);
- }
- Path file = Paths.get(args[0]);
-
- // (blockA, blockB) -> [totalBigrams, distinctSet via HashMap<Long, [count]>]
- // We use Maps of Maps to keep code simple; HAN is the only file
- // big enough to matter and fits in heap.
- Map<String, Map<Long, long[]>> byBlockPair = new HashMap<>();
- Map<String, long[]> blockPairTotals = new HashMap<>();
- long totalN = 0;
-
- try (BufferedReader r = new BufferedReader(
- new InputStreamReader(
- new GZIPInputStream(Files.newInputStream(file)),
- StandardCharsets.UTF_8))) {
- String line;
- while ((line = r.readLine()) != null) {
- int prevCp = -1;
- String prevBlock = null;
- for (int i = 0; i < line.length(); ) {
- int cp = line.codePointAt(i);
- i += Character.charCount(cp);
- String block = blockShortName(cp);
- if (prevCp >= 0) {
- String key = prevBlock + "|" + block;
- Map<Long, long[]> set = byBlockPair.computeIfAbsent(
- key, k -> new HashMap<>(256));
- long packed = ((long) prevCp << 24) | (cp & 0xFFFFFFL);
- long[] c = set.get(packed);
- if (c == null) {
- set.put(packed, new long[]{1L});
- } else {
- c[0]++;
- }
- blockPairTotals.computeIfAbsent(key, k -> new long[1])[0]++;
- totalN++;
- }
- prevCp = cp;
- prevBlock = block;
- }
- }
- }
-
- System.out.printf("File: %s%n", file);
- System.out.printf("Total bigram occurrences: %,d%n%n", totalN);
-
- // Sort block-pair keys by total occurrences (descending).
- List<Map.Entry<String, long[]>> sorted = new ArrayList<>(blockPairTotals.entrySet());
- sorted.sort(Comparator.comparingLong(
- (Map.Entry<String, long[]> e) -> -e.getValue()[0]));
-
- System.out.printf("%-50s %14s %14s %12s %8s%n",
- "block_pair", "occurrences", "distinct", "singletons", "%total");
- System.out.println(repeat('-', 105));
-
- long distinctTotal = 0;
- long singletonsTotal = 0;
- for (Map.Entry<String, long[]> e : sorted) {
- String pair = e.getKey();
- long n = e.getValue()[0];
- Map<Long, long[]> set = byBlockPair.get(pair);
- int distinct = set.size();
- int singletons = 0;
- for (long[] c : set.values()) {
- if (c[0] == 1) singletons++;
- }
- distinctTotal += distinct;
- singletonsTotal += singletons;
- double pct = 100.0 * n / totalN;
- if (pct < 0.1 && n < 1000) {
- continue; // skip tail noise rows
- }
- System.out.printf("%-50s %,14d %,14d %,12d %7.2f%%%n",
- pair, n, distinct, singletons, pct);
- }
- System.out.println(repeat('-', 105));
- System.out.printf("Total distinct pairs (incl. tail): %,d%n", distinctTotal);
- System.out.printf("Total singletons (incl. tail): %,d%n", singletonsTotal);
-
- // Roll up by individual block (left side only) to see per-block distinct counts.
- System.out.println();
- System.out.println("=== Per-leading-block roll-up ===");
- Map<String, Long> distinctByLeadingBlock = new HashMap<>();
- Map<String, Long> occByLeadingBlock = new HashMap<>();
- for (Map.Entry<String, Map<Long, long[]>> e : byBlockPair.entrySet()) {
- String leading = e.getKey().substring(0, e.getKey().indexOf('|'));
- distinctByLeadingBlock.merge(leading, (long) e.getValue().size(), Long::sum);
- long sum = 0;
- for (long[] c : e.getValue().values()) sum += c[0];
- occByLeadingBlock.merge(leading, sum, Long::sum);
- }
- List<Map.Entry<String, Long>> rollup = new ArrayList<>(occByLeadingBlock.entrySet());
- rollup.sort(Comparator.comparingLong(
- (Map.Entry<String, Long> e) -> -e.getValue()));
- System.out.printf("%-35s %14s %14s%n",
- "leading_block", "occurrences", "distinct(rough)");
- System.out.println(repeat('-', 70));
- for (Map.Entry<String, Long> e : rollup) {
- System.out.printf("%-35s %,14d %,14d%n",
- e.getKey(), e.getValue(),
- distinctByLeadingBlock.get(e.getKey()));
- }
- }
-
- /**
- * Short-name for the Unicode block containing {@code cp}. Compresses the
- * many CJK-related blocks into a handful of human-readable labels.
- *
- * <p>Splits ASCII into ASCII_DIGIT / ASCII_LETTER / ASCII_PUNCT so we can
- * distinguish numerals (which are content-bearing across all scripts) from
- * English-letter contamination and punctuation.
- */
- private static String blockShortName(int cp) {
- Character.UnicodeBlock b = Character.UnicodeBlock.of(cp);
- if (b == null) return "UNK";
-
- String name = b.toString();
- if (name.equals("BASIC_LATIN")) {
- if (cp >= '0' && cp <= '9') return "ASCII_DIGIT";
- if ((cp >= 'A' && cp <= 'Z') || (cp >= 'a' && cp <= 'z')) return "ASCII_LETTER";
- return "ASCII_PUNCT";
- }
- // Compress noisy block names for the report.
- if (name.startsWith("CJK_UNIFIED_IDEOGRAPHS_EXTENSION")) {
- return "CJK_EXT_" + name.substring(name.lastIndexOf('_') + 1);
- }
- if (name.equals("CJK_UNIFIED_IDEOGRAPHS")) return "CJK_UNIFIED";
- if (name.equals("CJK_SYMBOLS_AND_PUNCTUATION")) return "CJK_PUNCT";
- if (name.equals("CJK_COMPATIBILITY_IDEOGRAPHS")) return "CJK_COMPAT";
- if (name.equals("CJK_COMPATIBILITY_FORMS")) return "CJK_COMPAT_FORMS";
- if (name.equals("HALFWIDTH_AND_FULLWIDTH_FORMS")) return "HALF_FULL";
- if (name.equals("HIRAGANA")) return "HIRAGANA";
- if (name.equals("KATAKANA")) return "KATAKANA";
- if (name.equals("KATAKANA_PHONETIC_EXTENSIONS")) return "KATAKANA_EXT";
- if (name.equals("HANGUL_SYLLABLES")) return "HANGUL";
- if (name.equals("HANGUL_JAMO")) return "HANGUL_JAMO";
- if (name.equals("HANGUL_COMPATIBILITY_JAMO")) return "HANGUL_JAMO_C";
- if (name.equals("LATIN_1_SUPPLEMENT")) return "LATIN1";
- return name;
- }
-
- private static String repeat(char c, int n) {
- char[] buf = new char[n];
- java.util.Arrays.fill(buf, c);
- return new String(buf);
- }
-}
diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/BuildJunkTrainingData.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/BuildJunkTrainingData.java
index a80fafbd6b..a45f74a82c 100644
--- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/BuildJunkTrainingData.java
+++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/BuildJunkTrainingData.java
@@ -658,6 +658,13 @@ public class BuildJunkTrainingData {
if (text.indexOf('\uFFFD') >= 0) {
return null;
}
+ // NFC so the training tally matches JunkDetector.aggregate at
+ // inference time (which also NFC-normalises — see comment at
+ // JunkDetector#aggregate). Precomposed characters (Latin
+ // diacritics, Vietnamese, Indic combining-mark sequences) are
+ // stored as single codepoints, so bigram counts collapse mark
+ // + letter into one unit instead of splitting them — matching
+ // the natural NFC form of most source text.
text = Normalizer.normalize(text, Normalizer.Form.NFC);
if (text.getBytes(StandardCharsets.UTF_8).length < minBytes) {
return null;
diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/CountPerScriptBigrams.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/CountPerScriptBigrams.java
deleted file mode 100644
index b287012ddc..0000000000
--- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/CountPerScriptBigrams.java
+++ /dev/null
@@ -1,326 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.ml.junkdetect.tools;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Stream;
-import java.util.zip.GZIPInputStream;
-
-/**
- * Diagnostic tool for sizing a per-script F1 bigram store (v7 design).
- *
- * <p>Walks every {@code *.train.gz} in {@code dataDir}, treating each file as
- * one script's corpus. Counts (cpA, cpB) codepoint-pair frequencies and
- * reports, per script:
- *
- * <ul>
- * <li>total bigram occurrences (N)
- * <li>distinct pair count (U)
- * <li>singletons — pairs seen exactly once (these are usually the
- * worst candidates to keep; they often reflect OCR noise / rare
- * proper nouns and inflate U without helping discrimination)
- * <li>"effective" pair count = pairs seen at least {@code MIN_COUNT} times
- * <li>coverage curve: how many of the top-N most-frequent pairs are needed
- * to cover {x = 50, 75, 90, 95, 99, 99.9}% of all bigram occurrences
- * <li>estimated v7 model size for several candidate cutoffs, assuming
- * 2.25 bytes/pair (MPHF + 8-bit fingerprint + 8-bit value)
- * and 1.3 bytes/pair (MPHF + 8-bit value, no fingerprint)
- * </ul>
- *
- * <p>Usage:
- * <pre>
- * mvn -pl tika-ml/tika-ml-junkdetect exec:java \
- * -Dexec.mainClass=org.apache.tika.ml.junkdetect.tools.CountPerScriptBigrams \
- * -Dexec.args="/path/to/junkdetect"
- * </pre>
- *
- * <p>No model output; this is read-only telemetry to inform the v7 sizing
- * decision (see {@code 20260514-junk-retrain-v6.md}).
- */
-public final class CountPerScriptBigrams {
-
- private static final int[] COVERAGE_PCT = {50, 75, 90, 95, 99};
- private static final double[] COVERAGE_FRAC_HI = {0.999};
-
- /** Cutoffs reported in the size-estimate table. */
- private static final int[] MIN_COUNT_CUTOFFS = {1, 2, 3, 5, 10};
-
- /** Bytes per retained pair for each candidate storage scheme. */
- private static final double[] BYTES_PER_PAIR_SCHEMES = {1.3, 2.25, 6.25};
- private static final String[] SCHEME_NAMES = {
- "MPHF+val(1.3B)", "MPHF+fp+val(2.25B)", "open-addr+key(6.25B)"};
-
- private CountPerScriptBigrams() {}
-
- public static void main(String[] args) throws IOException {
- if (args.length < 1) {
- System.err.println(
- "Usage: CountPerScriptBigrams <dataDir> [topK-per-script]");
- System.exit(1);
- }
- Path dataDir = Paths.get(args[0]);
- int topK = args.length >= 2 ? Integer.parseInt(args[1]) : 0;
-
- List<Path> trainFiles = new ArrayList<>();
- try (Stream<Path> s = Files.list(dataDir)) {
- s.filter(p -> p.getFileName().toString().endsWith(".train.gz"))
- .sorted()
- .forEach(trainFiles::add);
- }
- if (trainFiles.isEmpty()) {
- System.err.println("ERROR: no *.train.gz files in " + dataDir);
- System.exit(1);
- }
-
- System.out.printf("Found %d *.train.gz files in %s%n%n",
- trainFiles.size(), dataDir);
- System.out.printf(
- "%-22s %12s %12s %12s %12s | %s%n",
- "script", "total_N", "distinct_U", "singletons",
- "U(>=10)", "coverage: pairs needed for [50,75,90,95,99,99.9]%");
- System.out.println(repeat('-', 140));
-
- long grandTotalN = 0;
- long grandTotalU = 0;
- long grandTotalUge2 = 0;
- long grandTotalUge10 = 0;
-
- // Per-script size accumulators for the global-size summary at the end.
- Map<String, long[]> perScriptStats = new HashMap<>();
-
- for (Path trainFile : trainFiles) {
- String fname = trainFile.getFileName().toString();
- String script = fname.substring(0, fname.length() - ".train.gz".length())
- .toUpperCase();
-
- HashMap<Long, long[]> pairCounts = new HashMap<>(1 << 16);
- long totalN = 0;
- try (BufferedReader r = new BufferedReader(
- new InputStreamReader(
- new GZIPInputStream(Files.newInputStream(trainFile)),
- StandardCharsets.UTF_8))) {
- String line;
- while ((line = r.readLine()) != null) {
- int prevCp = -1;
- for (int i = 0; i < line.length(); ) {
- int cp = line.codePointAt(i);
- i += Character.charCount(cp);
- if (prevCp >= 0) {
- long key = packPair(prevCp, cp);
- long[] c = pairCounts.get(key);
- if (c == null) {
- pairCounts.put(key, new long[]{1L});
- } else {
- c[0]++;
- }
- totalN++;
- }
- prevCp = cp;
- }
- }
- }
-
- int distinctU = pairCounts.size();
-
- long[] counts = new long[distinctU];
- int idx = 0;
- for (long[] c : pairCounts.values()) {
- counts[idx++] = c[0];
- }
- // Sort descending for coverage curve.
- java.util.Arrays.sort(counts);
- // Reverse in place.
- for (int i = 0, j = counts.length - 1; i < j; i++, j--) {
- long t = counts[i];
- counts[i] = counts[j];
- counts[j] = t;
- }
-
- int singletons = 0;
- int uGe2 = 0;
- int uGe10 = 0;
- for (long c : counts) {
- if (c == 1) singletons++;
- if (c >= 2) uGe2++;
- if (c >= 10) uGe10++;
- }
-
- // Coverage thresholds: minimum k such that sum(counts[0..k-1]) / N >= t.
- int[] coveragePairs = new int[COVERAGE_PCT.length + COVERAGE_FRAC_HI.length];
- double[] thresholds = new double[coveragePairs.length];
- for (int i = 0; i < COVERAGE_PCT.length; i++) {
- thresholds[i] = COVERAGE_PCT[i] / 100.0;
- }
- for (int i = 0; i < COVERAGE_FRAC_HI.length; i++) {
- thresholds[COVERAGE_PCT.length + i] = COVERAGE_FRAC_HI[i];
- }
- long running = 0;
- int tIdx = 0;
- for (int k = 0; k < counts.length && tIdx < thresholds.length; k++) {
- running += counts[k];
- while (tIdx < thresholds.length
- && (double) running / totalN >= thresholds[tIdx]) {
- coveragePairs[tIdx++] = k + 1;
- }
- }
- // Fill any unreached thresholds with U (means: never reached, took all).
- for (; tIdx < thresholds.length; tIdx++) {
- coveragePairs[tIdx] = distinctU;
- }
-
- StringBuilder cov = new StringBuilder();
- for (int i = 0; i < coveragePairs.length; i++) {
- if (i > 0) cov.append(", ");
- cov.append(String.format("%,d", coveragePairs[i]));
- }
-
- System.out.printf("%-22s %,12d %,12d %,12d %,12d | %s%n",
- script.toLowerCase(),
- totalN, distinctU, singletons, uGe10,
- cov.toString());
-
- // Per-script size table.
- if (topK > 0 || true) {
- long[] sizeStats = new long[
- 2 + MIN_COUNT_CUTOFFS.length + BYTES_PER_PAIR_SCHEMES.length];
- sizeStats[0] = totalN;
- sizeStats[1] = distinctU;
- for (int i = 0; i < MIN_COUNT_CUTOFFS.length; i++) {
- int minC = MIN_COUNT_CUTOFFS[i];
- int kept = 0;
- for (long c : counts) {
- if (c >= minC) kept++;
- else break;
- }
- sizeStats[2 + i] = kept;
- }
- perScriptStats.put(script.toLowerCase(), sizeStats);
- }
-
- // Per-script top-K dump if requested.
- if (topK > 0) {
- System.out.printf(" top %d pairs in %s:%n", topK, script.toLowerCase());
- List<Map.Entry<Long, long[]>> sorted = new ArrayList<>(pairCounts.entrySet());
- sorted.sort((a, b) -> Long.compare(b.getValue()[0], a.getValue()[0]));
- for (int i = 0; i < Math.min(topK, sorted.size()); i++) {
- Map.Entry<Long, long[]> e = sorted.get(i);
- long k = e.getKey();
- int cpA = (int) (k >>> 24);
- int cpB = (int) (k & 0xFFFFFFL);
- System.out.printf(" U+%04X U+%04X (%c %c) %,d%n",
- cpA, cpB,
- safePrint(cpA), safePrint(cpB),
- e.getValue()[0]);
- }
- }
-
- grandTotalN += totalN;
- grandTotalU += distinctU;
- grandTotalUge2 += uGe2;
- grandTotalUge10 += uGe10;
- }
-
- System.out.println(repeat('-', 140));
- System.out.printf("%-22s %,12d %,12d %12s %,12d%n%n",
- "TOTAL", grandTotalN, grandTotalU,
- "-", grandTotalUge10);
-
- // ------------------------------------------------------------------
- // Cutoff vs. model-size summary
- // ------------------------------------------------------------------
- System.out.println("=== Model-size estimates by min-count cutoff and storage scheme ===");
- System.out.println("(sum of retained pairs across all scripts × bytes-per-pair)");
- System.out.println();
- System.out.printf("%-12s", "cutoff");
- for (String name : SCHEME_NAMES) {
- System.out.printf(" %20s", name);
- }
- System.out.printf(" %20s%n", "retained_pairs");
- System.out.println(repeat('-', 12 + (SCHEME_NAMES.length + 1) * 21));
-
- for (int i = 0; i < MIN_COUNT_CUTOFFS.length; i++) {
- long retained = 0;
- for (long[] stats : perScriptStats.values()) {
- retained += stats[2 + i];
- }
- System.out.printf("min_count>=%-2d", MIN_COUNT_CUTOFFS[i]);
- for (double bpp : BYTES_PER_PAIR_SCHEMES) {
- double bytes = retained * bpp;
- System.out.printf(" %18s ", humanBytes(bytes));
- }
- System.out.printf(" %,20d%n", retained);
- }
-
- System.out.println();
- System.out.println("Per-script pair counts retained at each cutoff:");
- System.out.printf("%-22s", "script");
- for (int c : MIN_COUNT_CUTOFFS) {
- System.out.printf(" %12s", ">=" + c);
- }
- System.out.println();
- List<Map.Entry<String, long[]>> sortedScripts =
- new ArrayList<>(perScriptStats.entrySet());
- sortedScripts.sort(Comparator.comparingLong(
- (Map.Entry<String, long[]> e) -> -e.getValue()[1]));
- for (Map.Entry<String, long[]> e : sortedScripts) {
- System.out.printf("%-22s", e.getKey());
- for (int i = 0; i < MIN_COUNT_CUTOFFS.length; i++) {
- System.out.printf(" %,12d", e.getValue()[2 + i]);
- }
- System.out.println();
- }
- }
-
- /** Pack two codepoints (each up to 21 bits) into a single long. */
- private static long packPair(int cpA, int cpB) {
- return ((long) cpA << 24) | (cpB & 0xFFFFFFL);
- }
-
- private static char safePrint(int cp) {
- if (cp < 0x20 || cp == 0x7F || !Character.isDefined(cp)) {
- return '.';
- }
- if (Character.charCount(cp) != 1) {
- return '?';
- }
- return (char) cp;
- }
-
- private static String repeat(char c, int n) {
- char[] buf = new char[n];
- java.util.Arrays.fill(buf, c);
- return new String(buf);
- }
-
- private static String humanBytes(double bytes) {
- if (bytes < 1024) return String.format("%.0f B", bytes);
- if (bytes < 1024 * 1024) return String.format("%.1f KB", bytes / 1024.0);
- if (bytes < 1024L * 1024 * 1024) return String.format("%.2f MB", bytes / (1024.0 * 1024));
- return String.format("%.2f GB", bytes / (1024.0 * 1024 * 1024));
- }
-}
diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/EvalJunkDetector.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/EvalJunkDetector.java
deleted file mode 100644
index e0b4bc0ae1..0000000000
--- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/EvalJunkDetector.java
+++ /dev/null
@@ -1,777 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.ml.junkdetect.tools;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.nio.charset.UnsupportedCharsetException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.stream.Collectors;
-import java.util.zip.GZIPInputStream;
-
-import org.apache.tika.ml.junkdetect.JunkDetector;
-import org.apache.tika.quality.TextQualityComparison;
-import org.apache.tika.quality.TextQualityScore;
-
-/**
- * Ablation evaluation for the junk detector.
- *
- * <p>For each script's dev set, scores clean sentences alongside several corruption
- * modes at various injection rates and string lengths. Computes per-cell Cohen's d
- * (discrimination power) and TPR/FPR at a fixed z-score threshold.
- *
- * <p>Output files in {@code --output-dir}:
- * <ul>
- * <li><b>detail.tsv</b> — one row per (script, distortion, rate, length):
- * {@code script, distortion, param, length, n_clean, n_corrupt,
- * mean_clean_z, mean_corrupt_z, cohens_d, fpr, tpr}
- * <li><b>summary.tsv</b> — macro-averaged Cohen's d and FPR/TPR per
- * (distortion, rate, length) across all scripts.
- * <li><b>compare.tsv</b> — pairwise codec-comparison accuracy using the
- * {@link JunkDetector#compare} API, stratified by string length.
- * This is the primary metric for the charset-arbitration use case;
- * larger mean delta = better discrimination at that length.
- * </ul>
- *
- * <p><b>Why char-remap is not in summary.tsv:</b> The character-level wrong-codec
- * substitution (e.g. CP1252→CP1255, replacing umlauts with Hebrew letters) is added
- * to training at a 5% rate. At that rate it is too subtle to detect via the absolute
- * {@link JunkDetector#score} API — z-score distributions barely separate (Cohen's d ≈ 0).
- * The distortion trains the LR to distinguish subtly-wrong from correct decodings, which
- * only manifests as larger pairwise deltas in {@link JunkDetector#compare}. Measuring it
- * via summary.tsv would produce misleading d≈0 "failure" rows; see compare.tsv instead.
- *
- * <p>Cohen's d = (mean_clean_z − mean_corrupt_z) / pooled_std.
- * Higher d = better discrimination. FPR = fraction of clean text falsely flagged;
- * TPR = fraction of corrupted text correctly flagged. Both use threshold = −2.0.
- *
- * <p>To compare two model versions: run eval before and after, then diff the
- * summary and compare TSVs. The "macro_d" column in summary.tsv and the
- * "mean_delta" columns in compare.tsv are the headline metrics.
- *
- * <p>Usage:
- * <pre>
- * java EvalJunkDetector \
- * --model /path/to/junkdetect.bin (default: classpath)
- * --data-dir ~/datasets/madlad/junkdetect
- * --output-dir /path/to/results (default: data-dir/eval)
- * --split dev|test (default: dev)
- * --samples 200
- * --compare-n 200 (qualifying pairs per codec pair per length)
- * --seed 42
- * --lengths 5,9,15,30,50,100,200
- * --compare-lengths 5,9,15,30,50
- * --rates 0.01,0.05,0.10,0.25,0.50,0.90
- * --threshold -2.0
- * </pre>
- */
-public class EvalJunkDetector {
-
- public static void main(String[] args) throws Exception {
- // Defaults
- Path modelPath = null;
- Path dataDir = Paths.get(System.getProperty("user.home"),
- "datasets", "madlad", "junkdetect");
- Path outputDir = null;
- String split = "dev";
- int samplesPerCell = 200;
- int compareN = 200;
- long seed = 42L;
- int[] lengths = {5, 9, 15, 30, 50, 100, 200};
- int[] compareLengths = {5, 9, 15, 30, 50};
- double[] rates = {0.01, 0.05, 0.10, 0.25, 0.50, 0.90};
- float threshold = -2.0f;
-
- for (int i = 0; i < args.length; i++) {
- switch (args[i]) {
- case "--model":
- modelPath = Paths.get(args[++i]);
- break;
- case "--data-dir":
- dataDir = Paths.get(args[++i]);
- break;
- case "--output-dir":
- outputDir = Paths.get(args[++i]);
- break;
- case "--split":
- split = args[++i];
- if (!split.equals("dev") && !split.equals("test")) {
- System.err.println("--split must be 'dev' or 'test'");
- System.exit(1);
- }
- break;
- case "--samples":
- samplesPerCell = Integer.parseInt(args[++i]);
- break;
- case "--compare-n":
- compareN = Integer.parseInt(args[++i]);
- break;
- case "--seed":
- seed = Long.parseLong(args[++i]);
- break;
- case "--lengths":
- lengths = Arrays.stream(args[++i].split(","))
- .mapToInt(Integer::parseInt).toArray();
- break;
- case "--compare-lengths":
- compareLengths = Arrays.stream(args[++i].split(","))
- .mapToInt(Integer::parseInt).toArray();
- break;
- case "--rates":
- rates = Arrays.stream(args[++i].split(","))
- .mapToDouble(Double::parseDouble).toArray();
- break;
- case "--threshold":
- threshold = Float.parseFloat(args[++i]);
- break;
- default:
- System.err.println("Unknown argument: " + args[i]);
- System.exit(1);
- }
- }
-
- if (outputDir == null) {
- outputDir = dataDir.resolve("eval");
- }
- Files.createDirectories(outputDir);
-
- JunkDetector detector = modelPath != null
- ? JunkDetector.loadFromPath(modelPath)
- : JunkDetector.loadFromClasspath();
-
- System.err.println("=== EvalJunkDetector ===");
- System.err.println(" data-dir: " + dataDir);
- System.err.println(" output-dir: " + outputDir);
- System.err.println(" split: " + split
- + (split.equals("test") ? " [FINAL REPORTING MODE]" : ""));
- System.err.println(" scripts in model: " + detector.knownScripts().size());
- System.err.println(" threshold: " + threshold);
-
- // Build wrong-codec remap tables for char-remap distortion
- List<Map<Character, Character>> remapTables = new ArrayList<>();
- for (String[] pair : TrainJunkModel.WRONG_CODEC_PAIRS) {
- Map<Character, Character> table = TrainJunkModel.buildRemapTable(pair[0], pair[1]);
- if (!table.isEmpty()) remapTables.add(table);
- }
- System.err.println(" remap tables: " + remapTables.size());
-
- String suffix = "." + split + ".gz";
- List<Path> devFiles;
- try (var stream = Files.list(dataDir)) {
- devFiles = stream
- .filter(p -> p.getFileName().toString().endsWith(suffix))
- .sorted()
- .collect(Collectors.toList());
- }
-
- if (devFiles.isEmpty()) {
- System.err.println("ERROR: no *" + suffix + " files found in " + dataDir);
- System.exit(1);
- }
-
- Path detailPath = outputDir.resolve("detail.tsv");
- Path summaryPath = outputDir.resolve("summary.tsv");
- Path comparePath = outputDir.resolve("compare.tsv");
-
- List<Row> allRows = new ArrayList<>();
-
- try (PrintWriter detail = new PrintWriter(
- Files.newBufferedWriter(detailPath, StandardCharsets.UTF_8))) {
-
- detail.println("script\tdistortion\tparam\tlength"
- + "\tn_clean\tn_corrupt"
- + "\tmean_clean_z\tmean_corrupt_z"
- + "\tcohens_d\tfpr\ttpr");
-
- for (Path devFile : devFiles) {
- String filename = devFile.getFileName().toString();
- String script = filename
- .substring(0, filename.length() - suffix.length())
- .toUpperCase();
-
- System.err.printf("%n--- %s ---%n", script);
-
- List<String> sentences = loadSentences(devFile, samplesPerCell * 20);
- if (sentences.size() < 10) {
- System.err.printf(" Skipping — only %d sentences%n", sentences.size());
- continue;
- }
-
- for (int len : lengths) {
- List<Float> cleanZ = scoreClean(detector, sentences, len,
- samplesPerCell, new Random(seed));
-
- // --- injection ---
- for (double rate : rates) {
- List<Float> corruptZ = scoreWithInjection(detector, sentences, len,
- rate, samplesPerCell, new Random(seed + 1));
- Row row = new Row(script, "inject",
- String.format("%.2f", rate), len,
- cleanZ, corruptZ, threshold);
- allRows.add(row);
- detail.println(row.toTsv());
- }
-
- // --- codepoint reversal ---
- {
- List<Float> corruptZ = scoreReversed(detector, sentences, len,
- samplesPerCell, new Random(seed + 2));
- Row row = new Row(script, "char-reverse", "-", len,
- cleanZ, corruptZ, threshold);
- allRows.add(row);
- detail.println(row.toTsv());
- }
-
- // --- byte shuffle ---
- {
- List<Float> corruptZ = scoreShuffled(detector, sentences, len,
- samplesPerCell, new Random(seed + 3));
- Row row = new Row(script, "byte-shuffle", "-", len,
- cleanZ, corruptZ, threshold);
- allRows.add(row);
- detail.println(row.toTsv());
- }
-
- // --- wrong-codec: re-read UTF-8 bytes as ISO-8859-1 then re-encode ---
- {
- List<Float> corruptZ = scoreWrongCodec(detector, sentences, len,
- samplesPerCell, new Random(seed + 4));
- Row row = new Row(script, "wrong-codec", "latin1-as-utf8", len,
- cleanZ, corruptZ, threshold);
- allRows.add(row);
- detail.println(row.toTsv());
- }
-
- // --- byte-swap ---
- {
- List<Float> corruptZ = scoreByteSwapped(detector, sentences, len,
- samplesPerCell, new Random(seed + 5));
- Row row = new Row(script, "byte-swap", "-", len,
- cleanZ, corruptZ, threshold);
- allRows.add(row);
- detail.println(row.toTsv());
- }
-
- // char-remap distortion is evaluated only via compare.tsv (pairwise delta),
- // not via absolute score() — see class Javadoc for rationale.
-
- detail.flush();
- }
- }
- }
-
- writeSummary(summaryPath, allRows, lengths, rates, threshold);
- writeCompareEval(detector, dataDir, suffix, comparePath, compareN, compareLengths, seed);
-
- System.err.println("\nWrote " + detailPath);
- System.err.println("Wrote " + summaryPath);
- System.err.println("Wrote " + comparePath);
- System.err.println("Done.");
- }
-
- // -----------------------------------------------------------------------
- // Summary aggregation
- // -----------------------------------------------------------------------
-
- private static void writeSummary(Path summaryPath, List<Row> rows,
- int[] lengths, double[] rates, float threshold)
- throws IOException {
- try (PrintWriter out = new PrintWriter(
- Files.newBufferedWriter(summaryPath, StandardCharsets.UTF_8))) {
-
- out.println("distortion\tparam\tlength\tn_scripts"
- + "\tmacro_cohens_d\tmacro_fpr\tmacro_tpr");
-
- List<String[]> conditions = new ArrayList<>();
- for (double rate : rates) {
- conditions.add(new String[]{"inject", String.format("%.2f", rate)});
- }
- conditions.add(new String[]{"char-reverse", "-"});
- conditions.add(new String[]{"byte-shuffle", "-"});
- conditions.add(new String[]{"wrong-codec", "latin1-as-utf8"});
- conditions.add(new String[]{"byte-swap", "-"});
- // char-remap is intentionally excluded from summary: at 5% rate the character-level
- // wrong-codec substitution is too subtle to detect via absolute score() — both the
- // clean and corrupted strings score similarly. The right metric for char-remap is
- // compare.tsv (pairwise delta), where it shows up strongly. Including it here would
- // make d≈0 rows that look like failures but are actually expected.
-
- for (String[] cond : conditions) {
- String distortion = cond[0];
- String param = cond[1];
- for (int len : lengths) {
- List<Row> matching = rows.stream()
- .filter(r -> r.distortion.equals(distortion)
- && r.param.equals(param)
- && r.length == len)
- .collect(Collectors.toList());
- if (matching.isEmpty()) continue;
-
- double macroCohensD = matching.stream()
- .filter(r -> !Double.isNaN(r.cohensD))
- .mapToDouble(r -> r.cohensD)
- .average().orElse(Double.NaN);
- double macroFpr = matching.stream()
- .mapToDouble(r -> r.fpr)
- .average().orElse(Double.NaN);
- double macroTpr = matching.stream()
- .mapToDouble(r -> r.tpr)
- .average().orElse(Double.NaN);
-
- out.printf("%s\t%s\t%d\t%d\t%.3f\t%.3f\t%.3f%n",
- distortion, param, len, matching.size(),
- macroCohensD, macroFpr, macroTpr);
- }
- }
-
- double overallD = rows.stream()
- .filter(r -> !Double.isNaN(r.cohensD))
- .mapToDouble(r -> r.cohensD)
- .average().orElse(Double.NaN);
- double overallFpr = rows.stream()
- .mapToDouble(r -> r.fpr)
- .average().orElse(Double.NaN);
- double overallTpr = rows.stream()
- .mapToDouble(r -> r.tpr)
- .average().orElse(Double.NaN);
- out.println();
- out.printf("# OVERALL macro_cohens_d=%.3f macro_fpr=%.3f macro_tpr=%.3f%n",
- overallD, overallFpr, overallTpr);
-
- System.err.printf("%nOVERALL: macro_cohens_d=%.3f macro_fpr=%.3f macro_tpr=%.3f%n",
- overallD, overallFpr, overallTpr);
- }
- }
-
- // -----------------------------------------------------------------------
- // Compare eval — pairwise codec arbitration, stratified by string length
- // -----------------------------------------------------------------------
-
- /**
- * For each entry in {@link TrainJunkModel#WRONG_CODEC_PAIRS}, encodes sentences
- * from the appropriate script's dev file as the source charset, then calls
- * {@link JunkDetector#compare} with the correct decoding (A) vs the wrong
- * decoding (B). Reports accuracy (how often A wins) and mean/median delta
- * at each requested string length.
- *
- * <p>Mean delta is the headline metric: larger delta means the model more
- * confidently picks the correct decoding. At short lengths (5–9 bytes)
- * delta is expected to be small; at 50 bytes it should be decisive.
- */
- private static void writeCompareEval(JunkDetector detector,
- Path dataDir, String suffix,
- Path comparePath,
- int nPerCell, int[] lengths,
- long seed) throws IOException {
- try (PrintWriter out = new PrintWriter(
- Files.newBufferedWriter(comparePath, StandardCharsets.UTF_8))) {
-
- out.println("source_codec\twrong_codec\tlength"
- + "\tn_tested\taccuracy\tmean_delta\tmedian_delta\tn_no_diff");
-
- System.err.printf("%n--- compare() eval ---%n");
-
- for (String[] pair : TrainJunkModel.WRONG_CODEC_PAIRS) {
- String sourceCodec = pair[0];
- String wrongCodec = pair[1];
-
- Charset srcCharset, wrongCharset;
- try {
- srcCharset = Charset.forName(sourceCodec);
- wrongCharset = Charset.forName(wrongCodec);
- } catch (UnsupportedCharsetException e) {
- System.err.printf(" [%s→%s] charset unavailable, skipping%n",
- sourceCodec, wrongCodec);
- continue;
- }
-
- String script = codecToScript(sourceCodec);
- Path devFile = dataDir.resolve(script.toLowerCase() + suffix);
- if (!Files.exists(devFile)) {
- System.err.printf(" [%s→%s] no dev file for %s, skipping%n",
- sourceCodec, wrongCodec, script);
- continue;
- }
-
- // Load a large pool; we'll filter down per-length
- List<String> allSentences = loadSentences(devFile, nPerCell * 50);
-
- // Pre-filter: keep only sentences that roundtrip through sourceCodec
- // and produce at least one differing character vs wrongCodec.
- List<String[]> candidates = new ArrayList<>(); // {asSource, asWrong}
- for (String sentence : allSentences) {
- byte[] bytes = sentence.getBytes(srcCharset);
- String asSource = new String(bytes, srcCharset);
- if (!asSource.equals(sentence)) continue; // encoding lost data
- String asWrong = new String(bytes, wrongCharset);
- if (asSource.equals(asWrong)) continue; // no differentiating bytes
- candidates.add(new String[]{asSource, asWrong});
- }
-
- if (candidates.isEmpty()) {
- System.err.printf(" [%s→%s] no qualifying sentences%n",
- sourceCodec, wrongCodec);
- continue;
- }
-
- System.err.printf(" [%s→%s] %d candidates from %s%n",
- sourceCodec, wrongCodec, candidates.size(), script);
-
- for (int targetLen : lengths) {
- Random rng = new Random(seed);
- // Shuffle candidates for this length independently
- List<String[]> shuffled = new ArrayList<>(candidates);
- Collections.shuffle(shuffled, rng);
-
- List<Float> deltas = new ArrayList<>();
- int nCorrect = 0;
- int nNoDiff = 0;
-
- for (String[] cand : shuffled) {
- if (deltas.size() + nNoDiff >= nPerCell * 3 && deltas.size() >= nPerCell) {
- break;
- }
- String asSource = trimToLength(cand[0], targetLen);
- String asWrong = trimToLength(cand[1], targetLen);
-
- if (asSource.equals(asWrong)) {
- nNoDiff++;
- continue;
- }
- if (asSource.isEmpty() || asWrong.isEmpty()) continue;
-
- TextQualityComparison result = detector.compare(
- sourceCodec, asSource, wrongCodec, asWrong);
-
- deltas.add(result.delta());
- if (sourceCodec.equals(result.winner())) nCorrect++;
- }
-
- if (deltas.isEmpty()) continue;
-
- double accuracy = (double) nCorrect / deltas.size();
- double meanDelta = deltas.stream().mapToDouble(Float::floatValue).average().orElse(0);
- List<Float> sorted = new ArrayList<>(deltas);
- Collections.sort(sorted);
- float medianDelta = sorted.get(sorted.size() / 2);
-
- System.err.printf(" len=%3d n=%3d acc=%.3f mean_delta=%.3f median_delta=%.3f%n",
- targetLen, deltas.size(), accuracy, meanDelta, medianDelta);
-
- out.printf("%s\t%s\t%d\t%d\t%.3f\t%.3f\t%.3f\t%d%n",
- sourceCodec, wrongCodec, targetLen,
- deltas.size(), accuracy, meanDelta, medianDelta, nNoDiff);
- }
- out.flush();
- }
- }
- }
-
- /**
- * Returns which dev-file script to use for a given source codec.
- * CP1251 → CYRILLIC, CP1253 → GREEK, CP1255 → HEBREW, everything else → LATIN.
- */
- private static String codecToScript(String codec) {
- switch (codec.toLowerCase()) {
- case "windows-1251": return "CYRILLIC";
- case "windows-1253": return "GREEK";
- case "windows-1255": return "HEBREW";
- default: return "LATIN";
- }
- }
-
- /**
- * Trims a string to approximately {@code targetLen} UTF-8 bytes, aligned to
- * a codepoint boundary. Used to produce short-string variants for compare() testing.
- */
- private static String trimToLength(String s, int targetLen) {
- byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
- if (bytes.length <= targetLen) return s;
- int end = targetLen;
- while (end < bytes.length && (bytes[end] & 0xC0) == 0x80) end++;
- return new String(bytes, 0, end, StandardCharsets.UTF_8);
- }
-
- // -----------------------------------------------------------------------
- // Row (one evaluation cell)
- // -----------------------------------------------------------------------
-
- private static final class Row {
- final String script;
- final String distortion;
- final String param;
- final int length;
- final int nClean;
- final int nCorrupt;
- final double meanCleanZ;
- final double meanCorruptZ;
- final double cohensD;
- final double fpr;
- final double tpr;
-
- Row(String script, String distortion, String param, int length,
- List<Float> cleanZ, List<Float> corruptZ, float threshold) {
- this.script = script;
- this.distortion = distortion;
- this.param = param;
- this.length = length;
- this.nClean = cleanZ.size();
- this.nCorrupt = corruptZ.size();
- this.meanCleanZ = mean(cleanZ);
- this.meanCorruptZ = mean(corruptZ);
- this.cohensD = computeCohensD(cleanZ, corruptZ);
- this.fpr = fractionBelow(cleanZ, threshold);
- this.tpr = fractionBelow(corruptZ, threshold);
- }
-
- String toTsv() {
- return String.format("%s\t%s\t%s\t%d\t%d\t%d\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f",
- script, distortion, param, length,
- nClean, nCorrupt,
- meanCleanZ, meanCorruptZ,
- cohensD, fpr, tpr);
- }
- }
-
- // -----------------------------------------------------------------------
- // Statistics
- // -----------------------------------------------------------------------
-
- private static double computeCohensD(List<Float> clean, List<Float> corrupt) {
- if (clean.isEmpty() || corrupt.isEmpty()) return Double.NaN;
- double mc = mean(clean);
- double mj = mean(corrupt);
- double vc = variance(clean, mc);
- double vj = variance(corrupt, mj);
- double pooledStd = Math.sqrt((vc + vj) / 2.0);
- if (pooledStd < 1e-9) return Double.NaN;
- return (mc - mj) / pooledStd;
- }
-
- private static double mean(List<Float> xs) {
- return xs.stream().mapToDouble(Float::floatValue).average().orElse(0);
- }
-
- private static double variance(List<Float> xs, double mu) {
- return xs.stream().mapToDouble(x -> (x - mu) * (x - mu)).average().orElse(0);
- }
-
- private static double fractionBelow(List<Float> zs, float threshold) {
- if (zs.isEmpty()) return Double.NaN;
- long count = zs.stream().filter(z -> z < threshold).count();
- return (double) count / zs.size();
- }
-
- // -----------------------------------------------------------------------
- // Scoring helpers
- // -----------------------------------------------------------------------
-
- private static List<Float> scoreClean(JunkDetector detector, List<String> sentences,
- int targetLen, int n, Random rng) {
- List<Float> results = new ArrayList<>(n);
- for (int i = 0; i < n; i++) {
- String s = pickSubstring(sentences, targetLen, rng);
- TextQualityScore score = detector.score(s);
- if (!score.isUnknown()) results.add(score.getZScore());
- }
- return results;
- }
-
- private static List<Float> scoreWithInjection(JunkDetector detector,
- List<String> sentences, int targetLen,
- double rate, int n, Random rng) {
- List<Float> results = new ArrayList<>(n);
- for (int i = 0; i < n; i++) {
- String s = pickSubstring(sentences, targetLen, rng);
- byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
- injectRandomBytes(bytes, rate, rng);
- TextQualityScore score = detector.score(new String(bytes, StandardCharsets.ISO_8859_1));
- if (!score.isUnknown()) results.add(score.getZScore());
- }
- return results;
- }
-
- private static List<Float> scoreReversed(JunkDetector detector, List<String> sentences,
- int targetLen, int n, Random rng) {
- List<Float> results = new ArrayList<>(n);
- for (int i = 0; i < n; i++) {
- String s = reverseCodepoints(pickSubstring(sentences, targetLen, rng));
- TextQualityScore score = detector.score(s);
- if (!score.isUnknown()) results.add(score.getZScore());
- }
- return results;
- }
-
- private static List<Float> scoreShuffled(JunkDetector detector, List<String> sentences,
- int targetLen, int n, Random rng) {
- List<Float> results = new ArrayList<>(n);
- for (int i = 0; i < n; i++) {
- String s = pickSubstring(sentences, targetLen, rng);
- byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
- shuffleBytes(bytes, rng);
- TextQualityScore score = detector.score(new String(bytes, StandardCharsets.ISO_8859_1));
- if (!score.isUnknown()) results.add(score.getZScore());
- }
- return results;
- }
-
- private static List<Float> scoreWrongCodec(JunkDetector detector, List<String> sentences,
- int targetLen, int n, Random rng) {
- List<Float> results = new ArrayList<>(n);
- for (int i = 0; i < n; i++) {
- String s = pickSubstring(sentences, targetLen, rng);
- byte[] garbled = wrongCodecBytes(s.getBytes(StandardCharsets.UTF_8));
- TextQualityScore score = detector.score(new String(garbled, StandardCharsets.UTF_8));
- if (!score.isUnknown()) results.add(score.getZScore());
- }
- return results;
- }
-
- private static List<Float> scoreByteSwapped(JunkDetector detector, List<String> sentences,
- int targetLen, int n, Random rng) {
- List<Float> results = new ArrayList<>(n);
- for (int i = 0; i < n; i++) {
- String s = pickSubstring(sentences, targetLen, rng);
- byte[] swapped = swapByteOrder(s.getBytes(StandardCharsets.UTF_8));
- TextQualityScore score = detector.score(new String(swapped, StandardCharsets.ISO_8859_1));
- if (!score.isUnknown()) results.add(score.getZScore());
- }
- return results;
- }
-
- /**
- * Applies a randomly chosen wrong-codec character remap at {@code rate} to each
- * sample. Simulates real-world charset misdetection at the character level
- * (e.g. CP1252-encoded text decoded as CP1255, replacing umlauts with Hebrew letters).
- */
- private static List<Float> scoreWithRemap(JunkDetector detector, List<String> sentences,
- int targetLen,
- List<Map<Character, Character>> remapTables,
- double rate, int n, Random rng) {
- List<Float> results = new ArrayList<>(n);
- for (int i = 0; i < n; i++) {
- String s = pickSubstring(sentences, targetLen, rng);
- Map<Character, Character> table = remapTables.get(rng.nextInt(remapTables.size()));
- String corrupted = TrainJunkModel.wrongCodecRemap(s, table, rate, rng);
- TextQualityScore score = detector.score(corrupted);
- if (!score.isUnknown()) results.add(score.getZScore());
- }
- return results;
- }
-
- // -----------------------------------------------------------------------
- // Distortion primitives
- // -----------------------------------------------------------------------
-
- /**
- * Injects control characters (0x01–0x09) at the given rate.
- */
- static void injectRandomBytes(byte[] bytes, double rate, Random rng) {
- for (int i = 0; i < bytes.length; i++) {
- if (rng.nextDouble() < rate) {
- bytes[i] = (byte) (0x01 + rng.nextInt(9));
- }
- }
- }
-
- /**
- * Wrong-codec distortion: UTF-8 bytes re-interpreted as ISO-8859-1, then
- * re-encoded as UTF-8. Produces bogus two-byte sequences for any non-ASCII byte.
- */
- static byte[] wrongCodecBytes(byte[] utf8) {
- String misread = new String(utf8, StandardCharsets.ISO_8859_1);
- return misread.getBytes(StandardCharsets.UTF_8);
- }
-
- /**
- * Swaps each adjacent pair of bytes — simulates reading a 2-byte encoding
- * (UTF-16, CP932 two-byte sequences) with wrong byte order.
- */
- static byte[] swapByteOrder(byte[] bytes) {
- byte[] out = bytes.clone();
- for (int i = 0; i + 1 < out.length; i += 2) {
- byte tmp = out[i];
- out[i] = out[i + 1];
- out[i + 1] = tmp;
- }
- return out;
- }
-
- static void shuffleBytes(byte[] bytes, Random rng) {
- for (int i = bytes.length - 1; i > 0; i--) {
- int j = rng.nextInt(i + 1);
- byte tmp = bytes[i];
- bytes[i] = bytes[j];
- bytes[j] = tmp;
- }
- }
-
- static String reverseCodepoints(String s) {
- int[] codepoints = s.codePoints().toArray();
- for (int lo = 0, hi = codepoints.length - 1; lo < hi; lo++, hi--) {
- int tmp = codepoints[lo];
- codepoints[lo] = codepoints[hi];
- codepoints[hi] = tmp;
- }
- return new String(codepoints, 0, codepoints.length);
- }
-
- // -----------------------------------------------------------------------
- // Sentence sampling
- // -----------------------------------------------------------------------
-
- private static String pickSubstring(List<String> sentences, int targetLen, Random rng) {
- String s = sentences.get(rng.nextInt(sentences.size()));
- byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
- if (bytes.length <= targetLen) return s;
- int start = rng.nextInt(bytes.length - targetLen);
- while (start > 0 && (bytes[start] & 0xC0) == 0x80) start--;
- int end = Math.min(start + targetLen, bytes.length);
- while (end < bytes.length && (bytes[end] & 0xC0) == 0x80) end++;
- return new String(bytes, start, end - start, StandardCharsets.UTF_8);
- }
-
- private static List<String> loadSentences(Path devGz, int maxSentences) throws IOException {
- List<String> result = new ArrayList<>();
- try (BufferedReader r = new BufferedReader(
- new InputStreamReader(
- new GZIPInputStream(Files.newInputStream(devGz)),
- StandardCharsets.UTF_8))) {
- String line;
- while ((line = r.readLine()) != null && result.size() < maxSentences) {
- String trimmed = line.strip();
- if (!trimmed.isEmpty()
- && trimmed.getBytes(StandardCharsets.UTF_8).length >= 5) {
- result.add(trimmed);
- }
- }
- }
- return result;
- }
-}
diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/EvalJunkOnCharsetDevtest.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/EvalJunkOnCharsetDevtest.java
deleted file mode 100644
index 30d175a4b1..0000000000
--- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/EvalJunkOnCharsetDevtest.java
+++ /dev/null
@@ -1,688 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.ml.junkdetect.tools;
-
-import java.io.DataInputStream;
-import java.io.EOFException;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.nio.ByteBuffer;
-import java.nio.charset.CharacterCodingException;
-import java.nio.charset.Charset;
-import java.nio.charset.CharsetDecoder;
-import java.nio.charset.CodingErrorAction;
-import java.nio.charset.IllegalCharsetNameException;
-import java.nio.charset.StandardCharsets;
-import java.nio.charset.UnsupportedCharsetException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Stream;
-import java.util.zip.GZIPInputStream;
-
-import org.apache.tika.ml.junkdetect.JunkDetector;
-import org.apache.tika.quality.TextQualityScore;
-
-/**
- * Eval harness: for each labeled charset in {@code ~/data/charsets/devtest/},
- * decode under its true charset (clean) and under a curated set of wrong
- * charsets (mojibake), score with {@link JunkDetector}, report margin
- * statistics per (labeled_charset × wrong_charset × source-byte-length).
- *
- * <p>Devtest file format: gzip → repeated {@code [u16 big-endian length,
- * length bytes]} records, where the bytes are real text encoded in the
- * labeled charset. Same format the charset trainer consumes.
- *
- * <p>Output (TSVs):
- * <ul>
- * <li><b>detail.tsv</b>: one row per (labeled_cs, script, wrong_cs, length).
- * Columns: n, mean_clean_z, mean_mojibake_z, cohens_d, mean_margin,
- * p5_margin, p50_margin, fpr, tpr.</li>
- * <li><b>summary.tsv</b>: macro-averaged across wrong charsets, per
- * (script, length). The headline "is this script in trouble?" view.</li>
- * <li><b>script_pivot.tsv</b>: per-script rollup across all lengths +
- * wrong charsets. Single-number-per-script view for spot inversion.</li>
- * </ul>
- *
- * <p>"Margin" is the per-record paired difference {@code clean_z -
- * mojibake_z}. Mean margin and 5th-percentile margin are the
- * margin-maximization metrics the v6 retrain is optimizing for. Cohen's d
- * is the independent-distribution analog (kept for compatibility with the
- * existing {@link EvalJunkDetector} schema).
- *
- * <p>Usage:
- * <pre>
- * ./mvnw -pl tika-ml/tika-ml-junkdetect exec:java \
- * -Dexec.mainClass=org.apache.tika.ml.junkdetect.tools.EvalJunkOnCharsetDevtest \
- * -Dexec.args="--devtest-dir ~/data/charsets/devtest --output-dir /tmp/v5-baseline"
- * </pre>
- */
-public class EvalJunkOnCharsetDevtest {
-
- /**
- * Curated set of wrong charsets to cross-decode every labeled charset
- * against. Chosen to span the common real-world mojibake families:
- * Western Latin (cp1252, ISO-8859-1, MacRoman), CJK over-claim (GB18030,
- * Big5-HKSCS, Shift_JIS), Cyrillic (KOI8-R, cp1251), Arabic (cp1256),
- * EBCDIC over-claim (IBM424), DOS Latin (IBM850), and UTF-8 (catches
- * non-UTF8 bytes as replacement-character garbage).
- */
- private static final List<String> DEFAULT_WRONG_CHARSETS = List.of(
- "windows-1252", "ISO-8859-1", "x-MacRoman",
- "GB18030", "Big5-HKSCS", "Shift_JIS",
- "KOI8-R", "windows-1251",
- "windows-1256", "IBM424",
- "IBM850", "UTF-8"
- );
-
- /** Source-byte length buckets to slice records into. */
- private static final int[] DEFAULT_LENGTHS = {20, 50, 100, 200, 500, 1000};
-
- /** Cap on records loaded per labeled-charset file. */
- private static final int DEFAULT_MAX_RECORDS = 2000;
-
- /** Threshold for FPR/TPR reporting; matches EvalJunkDetector default. */
- private static final float DEFAULT_THRESHOLD = -2.0f;
-
- /** Minimum number of paired (clean, mojibake) samples per cell to emit a row. */
- private static final int MIN_SAMPLES_PER_CELL = 30;
-
- public static void main(String[] args) throws IOException {
- Path devtestDir = Paths.get(System.getProperty("user.home"),
- "data", "charsets", "devtest");
- Path outputDir = Paths.get("/tmp/junkdetect-eval");
- Path modelPath = null;
- int maxRecords = DEFAULT_MAX_RECORDS;
- int[] lengths = DEFAULT_LENGTHS;
- float threshold = DEFAULT_THRESHOLD;
- List<String> wrongCharsets = DEFAULT_WRONG_CHARSETS;
- List<String> labeledFilter = null;
-
- for (int i = 0; i < args.length; i++) {
- switch (args[i]) {
- case "--devtest-dir":
- devtestDir = Paths.get(args[++i]);
- break;
- case "--output-dir":
- outputDir = Paths.get(args[++i]);
- break;
- case "--model":
- modelPath = Paths.get(args[++i]);
- break;
- case "--max-records":
- maxRecords = Integer.parseInt(args[++i]);
- break;
- case "--threshold":
- threshold = Float.parseFloat(args[++i]);
- break;
- case "--lengths":
- lengths = Arrays.stream(args[++i].split(","))
- .mapToInt(Integer::parseInt).toArray();
- break;
- case "--wrong-charsets":
- wrongCharsets = Arrays.asList(args[++i].split(","));
- break;
- case "--only":
- labeledFilter = Arrays.asList(args[++i].split(","));
- break;
- default:
- System.err.println("Unknown arg: " + args[i]);
- printUsage();
- System.exit(1);
- }
- }
-
- if (!Files.isDirectory(devtestDir)) {
- System.err.println("ERROR: devtest-dir not found: " + devtestDir);
- System.exit(1);
- }
- Files.createDirectories(outputDir);
-
- JunkDetector detector = modelPath != null
- ? JunkDetector.loadFromPath(modelPath)
- : JunkDetector.loadFromClasspath();
-
- System.err.println("=== EvalJunkOnCharsetDevtest ===");
- System.err.println(" devtest-dir: " + devtestDir);
- System.err.println(" output-dir: " + outputDir);
- System.err.println(" model: " + (modelPath != null ? modelPath : "classpath default"));
- System.err.println(" model version: " + detector.getModelVersion());
- System.err.println(" max-records: " + maxRecords);
- System.err.println(" lengths: " + Arrays.toString(lengths));
- System.err.println(" threshold: " + threshold);
- System.err.println(" wrong-cs: " + wrongCharsets);
-
- // Resolve wrong charsets (skip any the JVM doesn't have)
- Map<String, Charset> resolvedWrong = new LinkedHashMap<>();
- for (String name : wrongCharsets) {
- Charset cs = tryGetCharset(name);
- if (cs == null) {
- System.err.println(" WARN: wrong-charset unavailable: " + name);
- continue;
- }
- resolvedWrong.put(name, cs);
- }
-
- List<Path> files;
- try (Stream<Path> stream = Files.list(devtestDir)) {
- files = stream
- .filter(p -> p.getFileName().toString().endsWith(".bin.gz"))
- .sorted()
- .toList();
- }
- if (files.isEmpty()) {
- System.err.println("ERROR: no *.bin.gz files in " + devtestDir);
- System.exit(1);
- }
-
- Path detailPath = outputDir.resolve("detail.tsv");
- Path summaryPath = outputDir.resolve("summary.tsv");
- Path pivotPath = outputDir.resolve("script_pivot.tsv");
-
- List<Row> allRows = new ArrayList<>();
-
- try (PrintWriter detail = new PrintWriter(
- Files.newBufferedWriter(detailPath, StandardCharsets.UTF_8))) {
-
- detail.println("labeled_cs\tscript\twrong_cs\tlength\tn"
- + "\tmean_clean_z\tmean_mojibake_z\tcohens_d"
- + "\tmean_margin\tp5_margin\tp50_margin"
- + "\tfpr\ttpr");
-
- for (Path file : files) {
- String labeledName = filenameToCharsetName(file);
- if (labeledFilter != null && !labeledFilter.contains(labeledName)) {
- continue;
- }
- Charset labeled = tryGetCharset(labeledName);
- if (labeled == null) {
- System.err.println(" SKIP: labeled charset unavailable: " + labeledName);
- continue;
- }
-
- List<byte[]> records = readRecords(file, maxRecords);
- if (records.size() < MIN_SAMPLES_PER_CELL) {
- System.err.printf(" SKIP %s: only %d records%n",
- labeledName, records.size());
- continue;
- }
-
- System.err.printf("%n--- %s (%d records) ---%n",
- labeledName, records.size());
-
- for (int len : lengths) {
- List<byte[]> slices = sliceToLength(records, len);
- if (slices.size() < MIN_SAMPLES_PER_CELL) {
- continue;
- }
-
- // Decode all slices under labeled (clean) once
- List<String> cleanTexts = decodeAll(slices, labeled);
- List<Float> cleanZs = scoreAll(detector, cleanTexts);
- if (cleanZs.size() < MIN_SAMPLES_PER_CELL) {
- continue;
- }
-
- // Detect script from a sample of the clean decoded text
- String script = detectDominantScript(
- cleanTexts.get(cleanTexts.size() / 2));
-
- for (Map.Entry<String, Charset> entry : resolvedWrong.entrySet()) {
- String wrongName = entry.getKey();
- Charset wrongCs = entry.getValue();
- if (equalCharset(labeled, wrongCs)) {
- continue; // can't be its own mojibake
- }
-
- List<String> mojiTexts = decodeAll(slices, wrongCs);
- // Pair cleanTexts[i] with mojiTexts[i] by source record
- Row row = scorePairs(detector, script, labeledName,
- wrongName, len, cleanTexts, mojiTexts,
- cleanZs, threshold);
- if (row == null) {
- continue;
- }
- allRows.add(row);
- detail.println(row.toTsv());
- }
- detail.flush();
- System.err.printf(" len=%4d n_clean=%d cells=%d%n",
- len, cleanZs.size(),
- allRows.stream()
- .filter(r -> r.labeledCs.equals(labeledName)
- && r.length == len)
- .count());
- }
- }
- }
-
- writeSummary(summaryPath, allRows, lengths);
- writeScriptPivot(pivotPath, allRows);
-
- System.err.println("\nWrote " + detailPath);
- System.err.println("Wrote " + summaryPath);
- System.err.println("Wrote " + pivotPath);
- System.err.println("Done.");
- }
-
- // -----------------------------------------------------------------------
- // Per-cell scoring (one labeled × wrong × length cell)
- // -----------------------------------------------------------------------
-
- private static Row scorePairs(JunkDetector detector,
- String script,
- String labeledName, String wrongName,
- int length,
- List<String> cleanTexts,
- List<String> mojiTexts,
- List<Float> cleanZsPre,
- float threshold) {
- // cleanZsPre is the already-scored clean text (avoid re-scoring per wrong cs).
- // We re-score only the mojibake side here.
- int n = Math.min(cleanTexts.size(), mojiTexts.size());
- List<Float> cleanZs = new ArrayList<>(n);
- List<Float> mojiZs = new ArrayList<>(n);
- List<Float> margins = new ArrayList<>(n);
- for (int i = 0; i < n; i++) {
- float cz = cleanZsPre.get(i);
- TextQualityScore ms = detector.score(mojiTexts.get(i));
- if (ms.isUnknown()) {
- continue;
- }
- float mz = ms.getZScore();
- cleanZs.add(cz);
- mojiZs.add(mz);
- margins.add(cz - mz);
- }
- if (margins.size() < MIN_SAMPLES_PER_CELL) {
- return null;
- }
- return new Row(labeledName, script, wrongName, length,
- cleanZs, mojiZs, margins, threshold);
- }
-
- // -----------------------------------------------------------------------
- // I/O: read the gzipped length-prefixed record format
- // -----------------------------------------------------------------------
-
- private static List<byte[]> readRecords(Path file, int maxRecords) throws IOException {
- List<byte[]> records = new ArrayList<>();
- try (FileInputStream fis = new FileInputStream(file.toFile());
- GZIPInputStream gis = new GZIPInputStream(fis);
- DataInputStream dis = new DataInputStream(gis)) {
- while (records.size() < maxRecords) {
- int len;
- try {
- len = dis.readUnsignedShort();
- } catch (EOFException eof) {
- break;
- }
- byte[] rec = new byte[len];
- dis.readFully(rec);
- records.add(rec);
- }
- }
- return records;
- }
-
- private static List<byte[]> sliceToLength(List<byte[]> records, int len) {
- List<byte[]> slices = new ArrayList<>();
- for (byte[] r : records) {
- if (r.length >= len) {
- slices.add(Arrays.copyOf(r, len));
- }
- }
- return slices;
- }
-
- private static List<String> decodeAll(List<byte[]> slices, Charset cs) {
- List<String> texts = new ArrayList<>(slices.size());
- for (byte[] s : slices) {
- texts.add(decode(s, cs));
- }
- return texts;
- }
-
- private static String decode(byte[] bytes, Charset cs) {
- CharsetDecoder dec = cs.newDecoder()
- .onMalformedInput(CodingErrorAction.REPLACE)
- .onUnmappableCharacter(CodingErrorAction.REPLACE);
- try {
- return dec.decode(ByteBuffer.wrap(bytes)).toString();
- } catch (CharacterCodingException e) {
- return new String(bytes, cs); // fallback; shouldn't happen with REPLACE
- }
- }
-
- private static List<Float> scoreAll(JunkDetector detector, List<String> texts) {
- List<Float> zs = new ArrayList<>(texts.size());
- for (String t : texts) {
- TextQualityScore s = detector.score(t);
- if (!s.isUnknown()) {
- zs.add(s.getZScore());
- } else {
- zs.add(Float.NaN);
- }
- }
- return zs;
- }
-
- // -----------------------------------------------------------------------
- // Aggregation: summary.tsv (macro across wrong charsets, per script×length)
- // -----------------------------------------------------------------------
-
- private static void writeSummary(Path summaryPath, List<Row> rows,
- int[] lengths) throws IOException {
- try (PrintWriter out = new PrintWriter(
- Files.newBufferedWriter(summaryPath, StandardCharsets.UTF_8))) {
- out.println("script\tlength\tn_cells"
- + "\tmacro_cohens_d\tmacro_mean_margin\tmacro_p5_margin"
- + "\tmacro_fpr\tmacro_tpr");
-
- // Group by (script, length)
- Map<String, Map<Integer, List<Row>>> bucketed = new HashMap<>();
- for (Row r : rows) {
- bucketed
- .computeIfAbsent(r.script, k -> new HashMap<>())
- .computeIfAbsent(r.length, k -> new ArrayList<>())
- .add(r);
- }
-
- List<String> scripts = new ArrayList<>(bucketed.keySet());
- Collections.sort(scripts);
- for (String script : scripts) {
- for (int len : lengths) {
- List<Row> cell = bucketed.get(script).get(len);
- if (cell == null || cell.isEmpty()) {
- continue;
- }
- double macroD = cell.stream()
- .filter(r -> !Double.isNaN(r.cohensD))
- .mapToDouble(r -> r.cohensD)
- .average().orElse(Double.NaN);
- double macroMargin = cell.stream()
- .mapToDouble(r -> r.meanMargin)
- .average().orElse(Double.NaN);
- double macroP5 = cell.stream()
- .mapToDouble(r -> r.p5Margin)
- .average().orElse(Double.NaN);
- double macroFpr = cell.stream()
- .mapToDouble(r -> r.fpr)
- .average().orElse(Double.NaN);
- double macroTpr = cell.stream()
- .mapToDouble(r -> r.tpr)
- .average().orElse(Double.NaN);
- out.printf("%s\t%d\t%d\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f%n",
- script, len, cell.size(),
- macroD, macroMargin, macroP5, macroFpr, macroTpr);
- }
- }
- }
- }
-
- // -----------------------------------------------------------------------
- // Aggregation: script_pivot.tsv (single line per script — quick triage)
- // -----------------------------------------------------------------------
-
- private static void writeScriptPivot(Path path, List<Row> rows) throws IOException {
- try (PrintWriter out = new PrintWriter(
- Files.newBufferedWriter(path, StandardCharsets.UTF_8))) {
- out.println("script\tn_cells"
- + "\tmean_d\tmean_margin\tmean_p5_margin"
- + "\tmin_d_cell\tmin_margin_cell");
-
- Map<String, List<Row>> byScript = new HashMap<>();
- for (Row r : rows) {
- byScript.computeIfAbsent(r.script, k -> new ArrayList<>()).add(r);
- }
- List<String> scripts = new ArrayList<>(byScript.keySet());
- Collections.sort(scripts);
- for (String script : scripts) {
- List<Row> cells = byScript.get(script);
- double meanD = cells.stream()
- .filter(r -> !Double.isNaN(r.cohensD))
- .mapToDouble(r -> r.cohensD)
- .average().orElse(Double.NaN);
- double meanMargin = cells.stream()
- .mapToDouble(r -> r.meanMargin)
- .average().orElse(Double.NaN);
- double meanP5 = cells.stream()
- .mapToDouble(r -> r.p5Margin)
- .average().orElse(Double.NaN);
- Row minDCell = cells.stream()
- .filter(r -> !Double.isNaN(r.cohensD))
- .min((a, b) -> Double.compare(a.cohensD, b.cohensD))
- .orElse(null);
- Row minMarginCell = cells.stream()
- .min((a, b) -> Double.compare(a.meanMargin, b.meanMargin))
- .orElse(null);
- out.printf("%s\t%d\t%.3f\t%.3f\t%.3f\t%s\t%s%n",
- script, cells.size(),
- meanD, meanMargin, meanP5,
- minDCell != null ? cellLabel(minDCell) : "-",
- minMarginCell != null ? cellLabel(minMarginCell) : "-");
- }
- }
- }
-
- private static String cellLabel(Row r) {
- return String.format("[%s→%s@%d]", r.labeledCs, r.wrongCs, r.length);
- }
-
- // -----------------------------------------------------------------------
- // Charset utilities
- // -----------------------------------------------------------------------
-
- private static String filenameToCharsetName(Path file) {
- String name = file.getFileName().toString();
- if (name.endsWith(".bin.gz")) {
- name = name.substring(0, name.length() - ".bin.gz".length());
- }
- return name;
- }
-
- private static Charset tryGetCharset(String name) {
- try {
- return Charset.forName(name);
- } catch (UnsupportedCharsetException | IllegalCharsetNameException e) {
- return null;
- }
- }
-
- private static boolean equalCharset(Charset a, Charset b) {
- return a.name().equalsIgnoreCase(b.name())
- || a.aliases().contains(b.name())
- || b.aliases().contains(a.name());
- }
-
- // -----------------------------------------------------------------------
- // Script detection (parallels JunkDetector.detectDominantScript, which is
- // package-private; small enough to inline)
- // -----------------------------------------------------------------------
-
- private static final Map<String, String> SCRIPT_FALLBACK = Map.of(
- "HIRAGANA", "HAN",
- "KATAKANA", "HAN"
- );
-
- private static String detectDominantScript(String text) {
- if (text == null || text.isEmpty()) {
- return "LATIN";
- }
- Map<Character.UnicodeScript, Integer> counts = new HashMap<>();
- for (int i = 0; i < text.length(); ) {
- int cp = text.codePointAt(i);
- Character.UnicodeScript s = Character.UnicodeScript.of(cp);
- if (s != Character.UnicodeScript.COMMON
- && s != Character.UnicodeScript.INHERITED
- && s != Character.UnicodeScript.UNKNOWN) {
- counts.merge(s, 1, Integer::sum);
- }
- i += Character.charCount(cp);
- }
- if (counts.isEmpty()) {
- return "LATIN";
- }
- String name = counts.entrySet().stream()
- .max(Map.Entry.comparingByValue())
- .map(e -> e.getKey().name())
- .orElse("LATIN");
- return SCRIPT_FALLBACK.getOrDefault(name, name);
- }
-
- // -----------------------------------------------------------------------
- // Row
- // -----------------------------------------------------------------------
-
- private static final class Row {
- final String labeledCs;
- final String script;
- final String wrongCs;
- final int length;
- final int n;
- final double meanCleanZ;
- final double meanMojiZ;
- final double cohensD;
- final double meanMargin;
- final double p5Margin;
- final double p50Margin;
- final double fpr;
- final double tpr;
-
- Row(String labeledCs, String script, String wrongCs, int length,
- List<Float> cleanZs, List<Float> mojiZs, List<Float> margins,
- float threshold) {
- this.labeledCs = labeledCs;
- this.script = script;
- this.wrongCs = wrongCs;
- this.length = length;
- this.n = margins.size();
- this.meanCleanZ = mean(cleanZs);
- this.meanMojiZ = mean(mojiZs);
- this.cohensD = computeCohensD(cleanZs, mojiZs);
- this.meanMargin = mean(margins);
- this.p5Margin = percentile(margins, 0.05);
- this.p50Margin = percentile(margins, 0.50);
- this.fpr = fractionBelow(cleanZs, threshold);
- this.tpr = fractionBelow(mojiZs, threshold);
- }
-
- String toTsv() {
- return String.format(
- "%s\t%s\t%s\t%d\t%d\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f",
- labeledCs, script, wrongCs, length, n,
- meanCleanZ, meanMojiZ, cohensD,
- meanMargin, p5Margin, p50Margin,
- fpr, tpr);
- }
- }
-
- // -----------------------------------------------------------------------
- // Statistics
- // -----------------------------------------------------------------------
-
- private static double computeCohensD(List<Float> a, List<Float> b) {
- if (a.size() < 2 || b.size() < 2) {
- return Double.NaN;
- }
- double ma = mean(a);
- double mb = mean(b);
- double va = variance(a, ma);
- double vb = variance(b, mb);
- double pooled = Math.sqrt((va + vb) / 2.0);
- if (pooled < 1e-9) {
- return Double.NaN;
- }
- return (ma - mb) / pooled;
- }
-
- private static double mean(List<Float> xs) {
- double s = 0;
- int n = 0;
- for (float f : xs) {
- if (!Float.isNaN(f)) {
- s += f;
- n++;
- }
- }
- return n == 0 ? Double.NaN : s / n;
- }
-
- private static double variance(List<Float> xs, double m) {
- if (xs.size() < 2) {
- return 0;
- }
- double s = 0;
- int n = 0;
- for (float f : xs) {
- if (!Float.isNaN(f)) {
- double d = f - m;
- s += d * d;
- n++;
- }
- }
- return n < 2 ? 0 : s / (n - 1);
- }
-
- private static double percentile(List<Float> xs, double p) {
- List<Float> sorted = new ArrayList<>(xs);
- sorted.removeIf(f -> Float.isNaN(f));
- if (sorted.isEmpty()) {
- return Double.NaN;
- }
- Collections.sort(sorted);
- int idx = (int) Math.floor(p * (sorted.size() - 1));
- return sorted.get(idx);
- }
-
- private static double fractionBelow(List<Float> xs, float threshold) {
- int below = 0;
- int n = 0;
- for (float f : xs) {
- if (!Float.isNaN(f)) {
- if (f < threshold) {
- below++;
- }
- n++;
- }
- }
- return n == 0 ? Double.NaN : (double) below / n;
- }
-
- // -----------------------------------------------------------------------
-
- private static void printUsage() {
- System.err.println("Usage:");
- System.err.println(" EvalJunkOnCharsetDevtest");
- System.err.println(" [--devtest-dir <path>] (default ~/data/charsets/devtest)");
- System.err.println(" [--output-dir <path>] (default /tmp/junkdetect-eval)");
- System.err.println(" [--model <path>] (default classpath junkdetect.bin)");
- System.err.println(" [--max-records N] (default 2000)");
- System.err.println(" [--threshold F] (default -2.0)");
- System.err.println(" [--lengths 20,50,...]");
- System.err.println(" [--wrong-charsets a,b,...]");
- System.err.println(" [--only labeledCs,...] (filter for spot runs)");
- }
-}
diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/JunkDetectorTrainingConfig.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/JunkDetectorTrainingConfig.java
index aa3761ef79..a3f801cc42 100644
--- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/JunkDetectorTrainingConfig.java
+++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/JunkDetectorTrainingConfig.java
@@ -44,9 +44,8 @@ import java.util.Set;
* model.
*
* <p>The constants below reflect the choices that produced the current
- * shipping model and are recorded in the corresponding training notes
- * ({@code 20260514-junk-retrain-v6.md}). Update them by editing this
- * file and committing the change together with the new model output.
+ * shipping model. Update them by editing this file and committing the
+ * change together with the new model output.
*
* <p>The class has no instance state; all values are exposed as
* {@code public static final}. This keeps callsites short and avoids
@@ -144,9 +143,7 @@ public final class JunkDetectorTrainingConfig {
* Per-script byte-budget overrides applied on top of the entropy-
* proportional allocation. Empty in the current configuration.
*
- * <p>Under v6 the {@code HAN=60MB} experiment <em>worsened</em> every
- * non-HAN script (the global F1 hash table was the bottleneck). Under
- * v7's per-script tables, the same experiment correctly leaves other
+ * <p>With per-script tables a {@code HAN=60MB} experiment leaves other
* scripts untouched, but the HAN gain itself was negligible (Cohen's d
* moved 7.26 → 7.35) — the per-script HAN model is already near its
* data-saturation point with ~18 MB of training data. Override left
diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/PrototypeCodepointHash.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/PrototypeCodepointHash.java
deleted file mode 100644
index 47ee346d37..0000000000
--- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/PrototypeCodepointHash.java
+++ /dev/null
@@ -1,1208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.ml.junkdetect.tools;
-
-import java.io.DataInputStream;
-import java.io.EOFException;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.nio.ByteBuffer;
-import java.nio.charset.CharacterCodingException;
-import java.nio.charset.Charset;
-import java.nio.charset.CharsetDecoder;
-import java.nio.charset.CodingErrorAction;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Stream;
-import java.util.zip.GZIPInputStream;
-
-import org.apache.tika.ml.chardetect.HtmlByteStripper;
-import org.apache.tika.ml.junkdetect.JunkDetector;
-import org.apache.tika.quality.TextQualityScore;
-
-/**
- * Throwaway prototype: validates the v6 codepoint-bigram-hash architecture
- * (Bloom-gated lookup with unigram backoff) by training on locally-available
- * text and measuring margins on the AIT5-class failure case (UTF-8 multi-
- * language records cross-decoded as GB18030).
- *
- * <p>Goal: prove the codepoint-bigram-hash approach opens the
- * UTF-8→GB18030 mojibake margin meaningfully above v5's ~1 z-unit
- * baseline BEFORE committing to a multi-day production retrain.
- *
- * <p>Training corpus: decode {@code ~/data/charsets/devtest/GB18030.bin.gz}
- * (Chinese) + first 80% of {@code UTF-8.bin.gz} (multi-language Wikipedia)
- * under their labeled charsets, iterate codepoints, count bigrams and unigrams,
- * hash into N buckets, build Bloom filter of seen pairs. Held-out: last 20%
- * of UTF-8 records.
- *
- * <p>Eval: for each held-out UTF-8 record, slice to length buckets
- * {20, 50, 100, 200, 500, 1000} source bytes. Decode each slice under
- * UTF-8 (clean) and GB18030 (mojibake-as-HAN). Score both with the
- * prototype model. Margin = clean_score - mojibake_score. Report
- * mean and 5th-percentile margin per length.
- *
- * <p>Sweep: {bigramBuckets, alpha} grid. Pick the configuration that
- * maximises margin. Compare to v5 baseline (mean margin ~1 z-unit
- * across all lengths in the same cohort).
- *
- * <p>Outputs:
- * <ul>
- * <li><b>prototype-sweep.tsv</b>: one row per
- * (bigram_buckets, alpha, length). Columns: n, mean_clean,
- * mean_moji, mean_margin, std_margin, p5_margin, p50_margin,
- * margin_in_clean_stds (effective z-units).</li>
- * </ul>
- *
- * <p>Usage:
- * <pre>
- * ./mvnw -pl tika-ml/tika-ml-junkdetect exec:java \
- * -Dexec.mainClass=org.apache.tika.ml.junkdetect.tools.PrototypeCodepointHash \
- * -Dexec.args="--devtest-dir ~/data/charsets/devtest --output-dir /tmp/v6-prototype"
- * </pre>
- */
-public class PrototypeCodepointHash {
-
- // --- Hyperparameter sweep grid ---
- private static final int[] BIGRAM_BUCKETS = {4096, 8192, 16384, 32768};
- private static final double[] ALPHAS = {1.0, 0.4};
- private static final int UNIGRAM_BUCKETS = 8192;
- private static final int BLOOM_BITS = 4 * 1024 * 1024; // 512 KB
- private static final int BLOOM_K = 7;
-
- // --- Smoothing ---
- private static final double ADD_ALPHA = 0.01;
-
- // --- Eval ---
- private static final int[] LENGTHS = {20, 50, 100, 200, 500, 1000};
- private static final int MAX_RECORDS_PER_FILE = 5000;
- private static final double HOLDOUT_FRACTION = 0.20;
- private static final int MIN_SCORE_CODEPOINTS = 3;
-
- public static void main(String[] args) throws IOException {
- Path devtestDir = Paths.get(System.getProperty("user.home"),
- "data", "charsets", "devtest");
- Path outputDir = Paths.get("/tmp/v6-prototype");
- int maxRecords = MAX_RECORDS_PER_FILE;
- List<Path> fixturesDirs = new ArrayList<>();
- String wrongCharsetName = "GB18030";
- boolean singleModel = false;
- List<String> candidates = List.of(
- "UTF-8", "GB18030", "windows-1252", "windows-1251", "windows-1257",
- "Shift_JIS", "EUC-JP", "ISO-2022-JP", "UTF-16LE", "UTF-16BE");
- List<String> forceCandidates = null; // when set, skip base detectors
- String expected = "UTF-8";
- int[] probeSizes = null; // when set, sweep these probe sizes per fixture
-
- for (int i = 0; i < args.length; i++) {
- switch (args[i]) {
- case "--devtest-dir":
- devtestDir = Paths.get(args[++i]);
- break;
- case "--output-dir":
- outputDir = Paths.get(args[++i]);
- break;
- case "--max-records":
- maxRecords = Integer.parseInt(args[++i]);
- break;
- case "--fixtures-dir":
- fixturesDirs.add(Paths.get(args[++i]));
- break;
- case "--wrong-charset":
- wrongCharsetName = args[++i];
- break;
- case "--single-model":
- // Skip prototype training; run N-way fixture eval on bundled JunkDetector only.
- singleModel = true;
- break;
- case "--candidates":
- candidates = Arrays.asList(args[++i].split(","));
- break;
- case "--force-candidates":
- // Bypass base detectors; pairwise tournament directly on these.
- forceCandidates = Arrays.asList(args[++i].split(","));
- break;
- case "--expected":
- expected = args[++i];
- break;
- case "--probe-sizes":
- // Comma-separated probe sizes (bytes). Each fixture
- // gets one row per size, so you can see how length
- // affects UNKNOWN vs scored.
- String[] sizes = args[++i].split(",");
- probeSizes = new int[sizes.length];
- for (int k = 0; k < sizes.length; k++) {
- probeSizes[k] = Integer.parseInt(sizes[k].trim());
- }
- break;
- default:
- System.err.println("Unknown arg: " + args[i]);
- System.exit(1);
- }
- }
- Files.createDirectories(outputDir);
-
- // --single-model bypasses the v5/v6-prototype comparison apparatus.
- // Requires --force-candidates to specify the charsets to compare;
- // the base-detector-driven path was removed to keep tika-ml-junkdetect
- // free of heavy encoding-detector deps.
- if (singleModel) {
- if (fixturesDirs.isEmpty()) {
- System.err.println("--single-model requires --fixtures-dir");
- System.exit(1);
- }
- if (forceCandidates == null || forceCandidates.isEmpty()) {
- System.err.println("--single-model requires --force-candidates "
- + "(e.g. --force-candidates UTF-8,GB18030)");
- System.exit(1);
- }
- evalFixturesSingleModel(fixturesDirs, forceCandidates, expected,
- probeSizes, outputDir);
- return;
- }
-
- System.err.println("=== PrototypeCodepointHash ===");
- System.err.println(" devtest-dir: " + devtestDir);
- System.err.println(" output-dir: " + outputDir);
- System.err.println(" max-records: " + maxRecords);
- System.err.println(" bigram_buckets sweep: " + Arrays.toString(BIGRAM_BUCKETS));
- System.err.println(" alpha sweep: " + Arrays.toString(ALPHAS));
- System.err.println(" unigram_buckets: " + UNIGRAM_BUCKETS);
- System.err.println(" bloom_bits: " + BLOOM_BITS
- + " (" + (BLOOM_BITS / 8 / 1024) + " KB, k=" + BLOOM_K + ")");
-
- // -------- Load corpus --------
-
- Charset utf8 = StandardCharsets.UTF_8;
- Charset gb18030 = Charset.forName("GB18030");
-
- System.err.println("\n--- Loading corpus ---");
- List<byte[]> utf8Records = readRecords(
- devtestDir.resolve("UTF-8.bin.gz"), maxRecords);
- List<byte[]> gbRecords = readRecords(
- devtestDir.resolve("GB18030.bin.gz"), maxRecords);
- System.err.printf(" UTF-8.bin.gz: %d records%n", utf8Records.size());
- System.err.printf(" GB18030.bin.gz: %d records%n", gbRecords.size());
-
- // Train/eval split on UTF-8 records. GB18030 records all go to training.
- int holdoutCount = (int) (utf8Records.size() * HOLDOUT_FRACTION);
- int utf8TrainSize = utf8Records.size() - holdoutCount;
- List<byte[]> utf8TrainBytes = utf8Records.subList(0, utf8TrainSize);
- List<byte[]> utf8EvalBytes = utf8Records.subList(utf8TrainSize, utf8Records.size());
- System.err.printf(" UTF-8 train: %d eval: %d%n",
- utf8TrainBytes.size(), utf8EvalBytes.size());
-
- // Decode training corpus to codepoint streams
- System.err.println("\n--- Decoding training corpus ---");
- List<int[]> trainStreams = new ArrayList<>();
- long totalTrainCp = 0;
- for (byte[] r : utf8TrainBytes) {
- int[] cps = toCodepoints(decode(r, utf8));
- if (cps.length >= 2) trainStreams.add(cps);
- totalTrainCp += cps.length;
- }
- for (byte[] r : gbRecords) {
- int[] cps = toCodepoints(decode(r, gb18030));
- if (cps.length >= 2) trainStreams.add(cps);
- totalTrainCp += cps.length;
- }
- System.err.printf(" total training codepoints: %,d across %d records%n",
- totalTrainCp, trainStreams.size());
-
- // Count unique pairs (for Bloom sizing sanity)
- Set<Long> uniquePairs = new HashSet<>();
- for (int[] cps : trainStreams) {
- for (int i = 0; i + 1 < cps.length; i++) {
- uniquePairs.add(packPair(cps[i], cps[i + 1]));
- if (uniquePairs.size() >= 2_000_000) break;
- }
- if (uniquePairs.size() >= 2_000_000) break;
- }
- System.err.printf(" unique codepoint-pairs in training: ~%,d%n",
- uniquePairs.size());
-
- // -------- Hyperparameter sweep --------
-
- Path sweepPath = outputDir.resolve("prototype-sweep.tsv");
- try (PrintWriter out = new PrintWriter(
- Files.newBufferedWriter(sweepPath, StandardCharsets.UTF_8))) {
- out.println("bigram_buckets\talpha\tlength\tn"
- + "\tmean_clean\tstd_clean\tmean_moji"
- + "\tmean_margin\tstd_margin\tp5_margin\tp50_margin"
- + "\tmargin_in_clean_stds\tbloom_seen_frac_clean\tbloom_seen_frac_moji");
-
- for (int buckets : BIGRAM_BUCKETS) {
- for (double alpha : ALPHAS) {
- System.err.printf("%n--- Config: bigram_buckets=%d alpha=%.1f ---%n",
- buckets, alpha);
-
- Model m = train(trainStreams, buckets, UNIGRAM_BUCKETS,
- BLOOM_BITS, BLOOM_K, ADD_ALPHA, alpha);
-
- // Calibrate on a sample of training streams (for the
- // "margin_in_clean_stds" effective-z normalization)
- double[] muSigma = calibrate(m, trainStreams);
- System.err.printf(" train mu=%.3f sigma=%.3f%n", muSigma[0], muSigma[1]);
-
- // Eval on held-out UTF-8 records
- for (int len : LENGTHS) {
- EvalCell cell = evalAtLength(m, utf8EvalBytes, len, utf8, gb18030);
- if (cell == null) continue;
- double effZ = cell.meanMargin / Math.max(muSigma[1], 1e-6);
- out.printf("%d\t%.2f\t%d\t%d\t%.4f\t%.4f\t%.4f\t%.4f\t%.4f\t%.4f\t%.4f\t%.3f\t%.3f\t%.3f%n",
- buckets, alpha, len, cell.n,
- cell.meanClean, cell.stdClean, cell.meanMoji,
- cell.meanMargin, cell.stdMargin,
- cell.p5Margin, cell.p50Margin,
- effZ, cell.bloomSeenFracClean, cell.bloomSeenFracMoji);
- System.err.printf(" len=%4d n=%-5d mean_margin=%6.3f p5=%6.3f"
- + " eff_z=%5.2f bloom_clean=%.2f bloom_moji=%.2f%n",
- len, cell.n, cell.meanMargin, cell.p5Margin, effZ,
- cell.bloomSeenFracClean, cell.bloomSeenFracMoji);
- out.flush();
- }
- }
- }
- }
- System.err.println("\nWrote " + sweepPath);
-
- // -------- Fixture eval (AIT5-class HTML files) --------
-
- if (!fixturesDirs.isEmpty()) {
- evalFixtures(trainStreams, fixturesDirs, wrongCharsetName, outputDir);
- }
-
- System.err.println("Done.");
- }
-
- // -----------------------------------------------------------------------
- // Real-life fixture eval: runs the production base detectors (BOM +
- // HtmlEncodingDetector + UniversalEncodingDetector) and asks the
- // JunkDetector to pick among their candidates via pairwise compare.
- // Mirrors the production charset-detection arbitration.
- // -----------------------------------------------------------------------
-
- private static void evalFixturesSingleModel(List<Path> fixturesDirs,
- List<String> forceCandidates,
- String expected,
- int[] probeSizes,
- Path outputDir) throws IOException {
- System.err.println("\n--- Forced-candidates fixture eval ---");
- System.err.println(" candidates: " + forceCandidates);
- JunkDetector detector = JunkDetector.loadFromClasspath();
- System.err.println(" model version: " + detector.getModelVersion());
- System.err.println(" expected: " + expected);
-
- List<Charset> forced = new ArrayList<>();
- for (String n : forceCandidates) {
- try {
- forced.add(Charset.forName(n));
- } catch (Exception e) {
- System.err.println(" skip unsupported charset: " + n);
- }
- }
-
- Path out = outputDir.resolve("fixtures-real-life.tsv");
- try (PrintWriter pw = new PrintWriter(
- Files.newBufferedWriter(out, StandardCharsets.UTF_8))) {
- pw.println("dir\tfile\tn_bytes\tprobe_size\texpected\tbom_cs\thtml_cs\tuniversal_cs"
- + "\tcandidates\twinner\tmargin\tstatus\tnotes");
- int pass = 0, fail = 0, skip = 0, agree = 0;
- double passMarginSum = 0.0;
- List<String> failingLines = new ArrayList<>();
-
- for (Path dir : fixturesDirs) {
- if (!Files.isDirectory(dir)) {
- System.err.println(" WARN: not a directory: " + dir);
- continue;
- }
- try (Stream<Path> stream = Files.walk(dir)) {
- List<Path> files = new ArrayList<>();
- stream.filter(Files::isRegularFile).forEach(files::add);
- Collections.sort(files);
- int[] sizes = probeSizes != null ? probeSizes : new int[]{16_384};
- for (Path f : files) {
- for (int sz : sizes) {
- FixtureResult r =
- evalOneForced(f, expected, detector, forced, sz);
- pw.println(r.toTsvLine());
- switch (r.status) {
- case "PASS":
- pass++;
- passMarginSum += r.margin;
- break;
- case "FAIL":
- fail++;
- failingLines.add(r.dir + "/" + r.shortName
- + "@" + sz + " -> " + r.winner
- + " (expected " + r.expected + ")");
- break;
- case "AGREE":
- agree++;
- break;
- default:
- skip++;
- }
- }
- }
- }
- }
- int n = pass + fail;
- System.err.println();
- System.err.println("=== Summary ===");
- System.err.printf("Pass: %d / %d (%.1f%%) — JunkDetector picked the expected charset%n",
- pass, n, n == 0 ? 0.0 : 100.0 * pass / n);
- System.err.printf("Fail: %d%n", fail);
- System.err.printf("Agree: %d (all detectors agreed; no arbitration needed)%n", agree);
- System.err.printf("Skip: %d%n", skip);
- if (pass > 0) {
- System.err.printf("Mean margin on pass: %.3f%n", passMarginSum / pass);
- }
- if (!failingLines.isEmpty()) {
- System.err.println("Failing:");
- Collections.sort(failingLines);
- for (String line : failingLines) {
- System.err.println(" " + line);
- }
- }
- }
- System.err.println("Wrote " + out);
- }
-
- private static FixtureResult evalOneForced(Path file, String expected,
- JunkDetector detector,
- List<Charset> forced,
- int probeBytes) throws IOException {
- byte[] raw = Files.readAllBytes(file);
- FixtureResult r = new FixtureResult();
- r.dir = file.getParent().getFileName().toString();
- String fname = file.getFileName().toString();
- r.shortName = fname.length() > 24 ? fname.substring(0, 24) : fname;
- r.bytes = raw.length;
- r.probeSize = probeBytes;
- r.expected = expected;
-
- if (isBinaryMagic(raw)) {
- r.status = "SKIP_BIN";
- return r;
- }
- // Strip HTML on the WHOLE raw buffer first, then slice to probeBytes
- // from the stripped content. Otherwise a small probe slice can land
- // entirely inside <!DOCTYPE>/<html>/<head> boilerplate and leave
- // nothing to score after strip.
- byte[] strippedFull = stripHtmlBytes(raw);
- byte[] forDecode = strippedFull.length > probeBytes
- ? Arrays.copyOf(strippedFull, probeBytes) : strippedFull;
- r.candidatesStr = forced.stream().map(Charset::name)
- .reduce((a, b) -> a + "," + b).orElse("-");
-
- // Always log every candidate in notes — even those JunkDetector
- // rejects as unknown — so the failure mode is visible. An
- // "unknown" score itself is meaningful information when the other
- // candidate scored fine.
- String winner = null;
- String runner = null;
- float winnerZ = Float.NEGATIVE_INFINITY;
- float runnerZ = Float.NEGATIVE_INFINITY;
- StringBuilder notes = new StringBuilder();
- int decoded_scored = 0;
- for (Charset cs : forced) {
- String decoded = applyEntityVariant(new String(forDecode, cs), "expanded");
- int cps = toCodepoints(decoded).length;
- if (cps < 3) {
- notes.append(cs.name()).append("=TOO_SHORT(").append(cps).append(") ");
- continue;
- }
- TextQualityScore s = detector.score(decoded);
- if (s.isUnknown()) {
- // Diagnose: is this script-not-in-model (neutral case) or
- // all-runs-fragmented-too-short (a real mojibake signal)?
- String why = diagnoseUnknown(decoded, detector);
- notes.append(cs.name()).append("=UNK[").append(why).append("] ");
- continue;
- }
- float z = s.getZScore();
- notes.append(cs.name()).append("=").append(String.format("%.2f", z)).append(" ");
- decoded_scored++;
- if (z > winnerZ) {
- runner = winner;
- runnerZ = winnerZ;
- winner = cs.name();
- winnerZ = z;
- } else if (z > runnerZ) {
- runner = cs.name();
- runnerZ = z;
- }
- }
- if (winner == null) {
- r.status = "NO_DECODE";
- r.notes = notes.toString().trim();
- return r;
- }
- r.winner = winner;
- if (decoded_scored < 2) {
- // Only one candidate scored; no real arbitration happened.
- r.margin = Float.NaN;
- r.status = safeCanonical(winner).equals(safeCanonical(expected))
- ? "ONLY_EXPECTED_SCORED" : "ONLY_WRONG_SCORED";
- } else {
- r.margin = winnerZ - runnerZ;
- r.status = safeCanonical(winner).equals(safeCanonical(expected)) ? "PASS" : "FAIL";
- }
- r.notes = notes.toString().trim();
- return r;
- }
-
- /**
- * Diagnose why JunkDetector returned UNKNOWN for {@code text}. Walks
- * the same script-run logic, then classifies the failure mode:
- * <ul>
- * <li>{@code EMPTY} — input had no characters.</li>
- * <li>{@code NO_MODELED_SCRIPT} — all runs are in scripts the model
- * doesn't know (legit reason to be neutral).</li>
- * <li>{@code ALL_RUNS_TOO_SHORT(N)} — runs exist in modeled scripts
- * but every one is <2 UTF-8 bytes. Strong mojibake signal —
- * text is a salad of single codepoints from many scripts.</li>
- * <li>{@code MIXED} — some runs were modeled-but-too-short and
- * some were unmodeled.</li>
- * </ul>
- */
- private static String diagnoseUnknown(String text, JunkDetector detector) {
- if (text == null || text.isEmpty()) {
- return "EMPTY";
- }
- Set<String> modeled = detector.knownScripts();
- // Walk codepoints, splitting on script boundaries — same as
- // JunkDetector.buildScriptRuns conceptually. Track per-script:
- // longest UTF-8-byte run length, plus a separate "unmodeled" tally.
- java.util.Map<String, Integer> longestModeled = new java.util.HashMap<>();
- int unmodeledRuns = 0;
- int modeledTooShortRuns = 0;
- int currentBytes = 0;
- String currentScript = null;
- for (int i = 0; i < text.length(); ) {
- int cp = text.codePointAt(i);
- int charCount = Character.charCount(cp);
- String script = Character.UnicodeScript.of(cp).name();
- // COMMON / INHERITED / UNKNOWN attach to preceding run, but for
- // diagnosis we don't need to be that precise — treat them as a
- // continuation.
- if ("COMMON".equals(script) || "INHERITED".equals(script)
- || "UNKNOWN".equals(script)) {
- if (currentScript != null) {
- currentBytes += new String(new int[]{cp}, 0, 1)
- .getBytes(StandardCharsets.UTF_8).length;
- }
- } else if (script.equals(currentScript)) {
- currentBytes += new String(new int[]{cp}, 0, 1)
- .getBytes(StandardCharsets.UTF_8).length;
- } else {
- // close out previous run
- tallyRun(currentScript, currentBytes, modeled, longestModeled);
- if (currentScript != null) {
- if (!modeled.contains(currentScript)) {
- unmodeledRuns++;
- } else if (currentBytes < 2) {
- modeledTooShortRuns++;
- }
- }
- currentScript = script;
- currentBytes = new String(new int[]{cp}, 0, 1)
- .getBytes(StandardCharsets.UTF_8).length;
- }
- i += charCount;
- }
- // close final run
- if (currentScript != null) {
- if (!modeled.contains(currentScript)) {
- unmodeledRuns++;
- } else if (currentBytes < 2) {
- modeledTooShortRuns++;
- } else {
- longestModeled.merge(currentScript, currentBytes, Math::max);
- }
- }
- boolean anyModeledLong = !longestModeled.isEmpty();
- if (anyModeledLong) {
- // Some modeled run is ≥2 bytes — shouldn't have hit UNKNOWN.
- // (Possible discrepancy with the production logic; reported as MIXED.)
- return "MIXED(modeled_long=" + longestModeled.size() + ")";
- }
- if (modeledTooShortRuns > 0 && unmodeledRuns > 0) {
- return "MIXED(short=" + modeledTooShortRuns
- + ",unmodeled=" + unmodeledRuns + ")";
- }
- if (modeledTooShortRuns > 0) {
- return "ALL_RUNS_TOO_SHORT(" + modeledTooShortRuns + ")";
- }
- if (unmodeledRuns > 0) {
- return "NO_MODELED_SCRIPT(" + unmodeledRuns + ")";
- }
- return "OTHER";
- }
-
- private static void tallyRun(String script, int bytes, Set<String> modeled,
- java.util.Map<String, Integer> longestModeled) {
- if (script == null) {
- return;
- }
- if (modeled.contains(script) && bytes >= 2) {
- longestModeled.merge(script, bytes, Math::max);
- }
- }
-
- /**
- * Run HtmlByteStripper over the entire input; return the stripped
- * content bytes (or the input verbatim if no tags found).
- */
- private static byte[] stripHtmlBytes(byte[] raw) {
- byte[] dst = new byte[raw.length];
- HtmlByteStripper.Result r =
- HtmlByteStripper.strip(raw, 0, raw.length, dst, 0);
- if (r.tagCount > 0 && r.length > 0) {
- return Arrays.copyOf(dst, r.length);
- }
- return raw;
- }
-
- private static boolean isBinaryMagic(byte[] b) {
- if (b.length < 4) {
- return false;
- }
- if (b[0] == 0x50 && b[1] == 0x4B
- && (b[2] == 0x03 || b[2] == 0x05 || b[2] == 0x07)) {
- return true; // ZIP / JAR / APK / docx
- }
- if ((b[0] & 0xFF) == 0x1F && (b[1] & 0xFF) == 0x8B) {
- return true; // gzip
- }
- if (b[0] == '%' && b[1] == 'P' && b[2] == 'D' && b[3] == 'F') {
- return true; // PDF
- }
- if ((b[0] & 0xFF) == 0xD0 && (b[1] & 0xFF) == 0xCF) {
- return true; // OLE2
- }
- return false;
- }
-
- private static String safeCanonical(String charset) {
- if (charset == null) {
- return "";
- }
- try {
- return Charset.forName(charset).name();
- } catch (Exception e) {
- return charset.toUpperCase();
- }
- }
-
- private static final class FixtureResult {
- String dir;
- String shortName;
- int bytes;
- int probeSize;
- String expected;
- String bomCs;
- String htmlCs;
- String universalCs;
- String candidatesStr = "-";
- String winner = "-";
- float margin = Float.NaN;
- String status = "";
- String notes = "";
-
- String toTsvLine() {
- return String.format("%s\t%s\t%d\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s",
- dir, shortName, bytes, probeSize, expected,
- str(bomCs), str(htmlCs), str(universalCs),
- candidatesStr, str(winner),
- Float.isNaN(margin) ? "-" : String.format("%.3f", margin),
- status, notes.isEmpty() ? "-" : notes);
- }
-
- private static String str(String s) {
- return s == null ? "-" : s;
- }
- }
-
- // -----------------------------------------------------------------------
- // Fixture eval: score real-world AIT5-class HTML files under v5 and v6
- // prototype, with byte-level HTML stripping and entity-variant comparison.
- // -----------------------------------------------------------------------
-
- private static void evalFixtures(List<int[]> trainStreams,
- List<Path> fixturesDirs,
- String wrongCharsetName,
- Path outputDir) throws IOException {
- System.err.println("\n--- Fixture eval (best config: 4096 buckets, alpha=1.0) ---");
- Model v6 = train(trainStreams, 4096, UNIGRAM_BUCKETS,
- BLOOM_BITS, BLOOM_K, ADD_ALPHA, 1.0);
- double[] muSigma = calibrate(v6, trainStreams);
- float mu = (float) muSigma[0];
- float sigma = (float) Math.max(muSigma[1], 1e-6);
- System.err.printf(" v6 train mu=%.3f sigma=%.3f%n", mu, sigma);
-
- JunkDetector v5 = JunkDetector.loadFromClasspath();
- Charset cleanCs = StandardCharsets.UTF_8;
- Charset wrongCs = Charset.forName(wrongCharsetName);
- System.err.println(" v5 model version: " + v5.getModelVersion());
- System.err.println(" clean charset: " + cleanCs.name());
- System.err.println(" mojibake charset: " + wrongCs.name());
-
- Path fixturesPath = outputDir.resolve("fixtures.tsv");
- try (PrintWriter out = new PrintWriter(
- Files.newBufferedWriter(fixturesPath, StandardCharsets.UTF_8))) {
- out.println("cluster\tfile\tentity_variant\tn_clean_cp\tn_moji_cp"
- + "\tv5_clean_z\tv5_moji_z\tv5_margin"
- + "\tv6_F1_clean\tv6_F1_moji\tv6_F1_margin"
- + "\tv6_combo_clean\tv6_combo_moji\tv6_combo_margin"
- + "\tdominant_script"
- + "\tv5_winner\tv6_F1_winner\tv6_combo_winner");
-
- for (Path dir : fixturesDirs) {
- if (!Files.isDirectory(dir)) {
- System.err.println(" WARN: not a directory: " + dir);
- continue;
- }
- try (java.util.stream.Stream<Path> files = Files.walk(dir)) {
- List<Path> sorted = new ArrayList<>();
- files.filter(Files::isRegularFile).forEach(sorted::add);
- Collections.sort(sorted);
- for (Path f : sorted) {
- evalOneFixture(f, v6, mu, sigma, v5, cleanCs, wrongCs, out);
- }
- }
- }
- }
- System.err.println("Wrote " + fixturesPath);
- }
-
- private static void evalOneFixture(Path file, Model v6, float v6Mu, float v6Sigma,
- JunkDetector v5,
- Charset cleanCs, Charset wrongCs,
- PrintWriter out) throws IOException {
- byte[] rawBytes = Files.readAllBytes(file);
- if (rawBytes.length > 16384) {
- rawBytes = Arrays.copyOf(rawBytes, 16384);
- }
- // Byte-level HTML strip (matches JunkFilterEncodingDetector production pipeline)
- byte[] stripDst = new byte[rawBytes.length];
- HtmlByteStripper.Result strip =
- HtmlByteStripper.strip(rawBytes, 0, rawBytes.length, stripDst, 0);
- byte[] forDecode = rawBytes;
- if (strip.tagCount > 0 && strip.length > 0) {
- forDecode = new byte[strip.length];
- System.arraycopy(stripDst, 0, forDecode, 0, strip.length);
- }
-
- String cluster = file.getParent().getFileName().toString();
- String fname = file.getFileName().toString();
- // shorten long content-hash names for readability in output
- String shortName = fname.length() > 12 ? fname.substring(0, 12) : fname;
-
- String cleanRaw = decode(forDecode, cleanCs);
- String mojiRaw = decode(forDecode, wrongCs);
-
- for (String variant : List.of("raw", "expanded", "removed")) {
- String clean = applyEntityVariant(cleanRaw, variant);
- String moji = applyEntityVariant(mojiRaw, variant);
- int[] cleanCps = toCodepoints(clean);
- int[] mojiCps = toCodepoints(moji);
- if (cleanCps.length < 3 || mojiCps.length < 3) continue;
-
- // --- v5 full pipeline (existing) ---
- TextQualityScore v5cs = v5.score(clean);
- TextQualityScore v5ms = v5.score(moji);
- float v5cleanZ = v5cs.isUnknown() ? Float.NaN : v5cs.getZScore();
- float v5mojiZ = v5ms.isUnknown() ? Float.NaN : v5ms.getZScore();
- float v5Margin = v5cleanZ - v5mojiZ;
-
- // --- v6 Feature 1 alone (codepoint-bigram-hash + Bloom + unigram backoff) ---
- ScoreResult v6c = score(v6, cleanCps);
- ScoreResult v6m = score(v6, mojiCps);
- double v6Margin = v6c.meanLogP - v6m.meanLogP;
-
- // --- v6 combined: substitute v6's F1 z-score into v5's classifier ---
- JunkDetector.FeatureComponents cleanFc = v5.scoreWithFeatureComponents(clean);
- JunkDetector.FeatureComponents mojiFc = v5.scoreWithFeatureComponents(moji);
- float v6F1zClean = (float) (v6c.meanLogP - v6Mu) / v6Sigma;
- float v6F1zMoji = (float) (v6m.meanLogP - v6Mu) / v6Sigma;
- float comboClean = recombineLogit(v6F1zClean, cleanFc);
- float comboMoji = recombineLogit(v6F1zMoji, mojiFc);
- float comboMargin = comboClean - comboMoji;
- String dominantScript = cleanFc != null ? cleanFc.dominantScript : "?";
-
- String v5Winner = Float.isNaN(v5Margin) ? "?" : (v5Margin > 0 ? "CLEAN" : "MOJI");
- String v6F1Winner = Double.isNaN(v6Margin) ? "?" : (v6Margin > 0 ? "CLEAN" : "MOJI");
- String v6cWinner = Float.isNaN(comboMargin) ? "?" : (comboMargin > 0 ? "CLEAN" : "MOJI");
-
- out.printf("%s\t%s\t%s\t%d\t%d"
- + "\t%.3f\t%.3f\t%.3f"
- + "\t%.4f\t%.4f\t%.4f"
- + "\t%.3f\t%.3f\t%.3f"
- + "\t%s\t%s\t%s\t%s%n",
- cluster, shortName, variant,
- cleanCps.length, mojiCps.length,
- v5cleanZ, v5mojiZ, v5Margin,
- v6c.meanLogP, v6m.meanLogP, v6Margin,
- comboClean, comboMoji, comboMargin,
- dominantScript,
- v5Winner, v6F1Winner, v6cWinner);
- out.flush();
- System.err.printf(" [%s/%s %-8s] v5: Δ%+6.2f %s v6F1: Δ%+6.3f %s v6combo: Δ%+6.2f %s script=%s%n",
- cluster, shortName, variant,
- v5Margin, v5Winner,
- v6Margin, v6F1Winner,
- comboMargin, v6cWinner,
- dominantScript);
- }
- }
-
- /**
- * Recomputes v5's per-script classifier logit with v6's F1 z-score
- * substituted for v5's z1. Approximation: keeps v5's classifier weights
- * (w1..w4, bias) which were trained on the OLD F1 distribution. A true
- * v6 retrain would re-fit w1 on the new F1 distribution; this version
- * gives a directional estimate of "what if we just swap F1?"
- */
- private static float recombineLogit(float v6F1z, JunkDetector.FeatureComponents fc) {
- if (fc == null || fc.classifierWeights == null) {
- return Float.NaN;
- }
- float[] cw = fc.classifierWeights;
- int nFeat = cw.length - 1;
- float logit = cw[nFeat]; // bias
- if (nFeat >= 1) logit += cw[0] * v6F1z;
- if (nFeat >= 2) logit += cw[1] * fc.z2;
- if (nFeat >= 3) logit += cw[2] * fc.z3;
- if (nFeat >= 4) logit += cw[3] * fc.z4;
- return logit;
- }
-
- // -----------------------------------------------------------------------
- // HTML entity expansion / removal (regex-based, sufficient for fixtures)
- // -----------------------------------------------------------------------
-
- private static final Pattern NUM_DEC = Pattern.compile("&#(\\d{1,7});");
- private static final Pattern NUM_HEX = Pattern.compile("&#[xX]([0-9a-fA-F]{1,6});");
- private static final Pattern NAMED =
- Pattern.compile("&(amp|lt|gt|quot|apos|nbsp|copy|reg);");
-
- private static String applyEntityVariant(String s, String variant) {
- switch (variant) {
- case "raw": return s;
- case "expanded": return expandEntities(s);
- case "removed": return removeEntities(s);
- default: throw new IllegalArgumentException(variant);
- }
- }
-
- private static String expandEntities(String in) {
- String s = in;
- s = NUM_DEC.matcher(s).replaceAll(mr -> {
- try {
- int cp = Integer.parseInt(mr.group(1));
- if (cp >= 0 && cp <= 0x10FFFF) {
- return Matcher.quoteReplacement(new String(Character.toChars(cp)));
- }
- } catch (NumberFormatException ignored) {
- // fall through, leave unchanged
- }
- return Matcher.quoteReplacement(mr.group());
- });
- s = NUM_HEX.matcher(s).replaceAll(mr -> {
- try {
- int cp = Integer.parseInt(mr.group(1), 16);
- if (cp >= 0 && cp <= 0x10FFFF) {
- return Matcher.quoteReplacement(new String(Character.toChars(cp)));
- }
- } catch (NumberFormatException ignored) {
- // fall through, leave unchanged
- }
- return Matcher.quoteReplacement(mr.group());
- });
- s = NAMED.matcher(s).replaceAll(mr -> {
- switch (mr.group(1)) {
- case "amp": return "&";
- case "lt": return "<";
- case "gt": return ">";
- case "quot": return "\"";
- case "apos": return "'";
- case "nbsp": return " ";
- case "copy": return "©";
- case "reg": return "®";
- default: return Matcher.quoteReplacement(mr.group());
- }
- });
- return s;
- }
-
- private static String removeEntities(String s) {
- s = NUM_DEC.matcher(s).replaceAll("");
- s = NUM_HEX.matcher(s).replaceAll("");
- s = NAMED.matcher(s).replaceAll("");
- return s;
- }
-
- // -----------------------------------------------------------------------
- // Training
- // -----------------------------------------------------------------------
-
- private static Model train(List<int[]> streams,
- int bigramBuckets, int unigramBuckets,
- int bloomBits, int bloomK,
- double addAlpha, double backoffAlpha) {
- if (Integer.bitCount(bigramBuckets) != 1 || Integer.bitCount(unigramBuckets) != 1) {
- throw new IllegalArgumentException("Bucket counts must be powers of 2");
- }
- long[] bigramCounts = new long[bigramBuckets];
- long[] unigramCounts = new long[unigramBuckets];
- long bigramTotal = 0;
- long unigramTotal = 0;
- long[] bloomBitArr = new long[(bloomBits + 63) / 64];
-
- for (int[] cps : streams) {
- for (int i = 0; i < cps.length; i++) {
- int cp = cps[i];
- int uBucket = (int) (fnv1aUnigram(cp) & (unigramBuckets - 1));
- unigramCounts[uBucket]++;
- unigramTotal++;
- if (i + 1 < cps.length) {
- int cpNext = cps[i + 1];
- int bBucket = (int) (fnv1aBigram(cp, cpNext) & (bigramBuckets - 1));
- bigramCounts[bBucket]++;
- bigramTotal++;
- bloomAdd(bloomBitArr, bloomBits, bloomK, cp, cpNext);
- }
- }
- }
-
- // Convert to log-probabilities with add-alpha smoothing
- float[] bigramLogP = new float[bigramBuckets];
- double bigramDenom = bigramTotal + addAlpha * bigramBuckets;
- for (int i = 0; i < bigramBuckets; i++) {
- double p = (bigramCounts[i] + addAlpha) / bigramDenom;
- bigramLogP[i] = (float) Math.log(p);
- }
- float[] unigramLogP = new float[unigramBuckets];
- double unigramDenom = unigramTotal + addAlpha * unigramBuckets;
- for (int i = 0; i < unigramBuckets; i++) {
- double p = (unigramCounts[i] + addAlpha) / unigramDenom;
- unigramLogP[i] = (float) Math.log(p);
- }
-
- return new Model(bigramBuckets, unigramBuckets, bigramLogP, unigramLogP,
- bloomBitArr, bloomBits, bloomK, backoffAlpha);
- }
-
- private static double[] calibrate(Model m, List<int[]> streams) {
- double s = 0;
- double s2 = 0;
- int n = 0;
- // Use a stride to avoid scoring every single train record
- int stride = Math.max(1, streams.size() / 1000);
- for (int i = 0; i < streams.size(); i += stride) {
- int[] cps = streams.get(i);
- if (cps.length < MIN_SCORE_CODEPOINTS) continue;
- ScoreResult r = score(m, cps);
- s += r.meanLogP;
- s2 += r.meanLogP * r.meanLogP;
- n++;
- }
- if (n == 0) return new double[]{0, 1};
- double mu = s / n;
- double var = Math.max(0, s2 / n - mu * mu);
- double sigma = Math.sqrt(var);
- return new double[]{mu, sigma};
- }
-
- // -----------------------------------------------------------------------
- // Scoring
- // -----------------------------------------------------------------------
-
- private static ScoreResult score(Model m, int[] cps) {
- if (cps.length < 2) return new ScoreResult(Double.NaN, 0, 0);
- double sum = 0;
- int n = 0;
- int seen = 0;
- for (int i = 0; i + 1 < cps.length; i++) {
- int cp1 = cps[i];
- int cp2 = cps[i + 1];
- double logP;
- if (bloomContains(m.bloomBits, m.bloomBitCount, m.bloomK, cp1, cp2)) {
- int b = (int) (fnv1aBigram(cp1, cp2) & (m.bigramBuckets - 1));
- logP = m.bigramLogP[b];
- seen++;
- } else {
- int u1 = (int) (fnv1aUnigram(cp1) & (m.unigramBuckets - 1));
- int u2 = (int) (fnv1aUnigram(cp2) & (m.unigramBuckets - 1));
- logP = m.backoffAlpha * (m.unigramLogP[u1] + m.unigramLogP[u2]);
- }
- sum += logP;
- n++;
- }
- return new ScoreResult(sum / n, n, seen);
- }
-
- private static final class ScoreResult {
- final double meanLogP;
- final int nPairs;
- final int seenPairs;
- ScoreResult(double m, int n, int s) {
- this.meanLogP = m;
- this.nPairs = n;
- this.seenPairs = s;
- }
- }
-
- // -----------------------------------------------------------------------
- // Eval at one length bucket
- // -----------------------------------------------------------------------
-
- private static EvalCell evalAtLength(Model m, List<byte[]> evalBytes, int length,
- Charset cleanCs, Charset wrongCs) {
- List<Double> cleans = new ArrayList<>();
- List<Double> mojis = new ArrayList<>();
- List<Double> margins = new ArrayList<>();
- double seenSumClean = 0, seenSumMoji = 0;
- int nSeenObs = 0;
- for (byte[] rec : evalBytes) {
- if (rec.length < length) continue;
- byte[] slice = Arrays.copyOf(rec, length);
- int[] cleanCps = toCodepoints(decode(slice, cleanCs));
- int[] mojiCps = toCodepoints(decode(slice, wrongCs));
- if (cleanCps.length < MIN_SCORE_CODEPOINTS
- || mojiCps.length < MIN_SCORE_CODEPOINTS) continue;
- ScoreResult sc = score(m, cleanCps);
- ScoreResult sm = score(m, mojiCps);
- if (Double.isNaN(sc.meanLogP) || Double.isNaN(sm.meanLogP)) continue;
- cleans.add(sc.meanLogP);
- mojis.add(sm.meanLogP);
- margins.add(sc.meanLogP - sm.meanLogP);
- if (sc.nPairs > 0) seenSumClean += (double) sc.seenPairs / sc.nPairs;
- if (sm.nPairs > 0) seenSumMoji += (double) sm.seenPairs / sm.nPairs;
- nSeenObs++;
- }
- if (margins.size() < 30) return null;
- EvalCell cell = new EvalCell();
- cell.n = margins.size();
- cell.meanClean = mean(cleans);
- cell.stdClean = std(cleans, cell.meanClean);
- cell.meanMoji = mean(mojis);
- cell.meanMargin = mean(margins);
- cell.stdMargin = std(margins, cell.meanMargin);
- cell.p5Margin = percentile(margins, 0.05);
- cell.p50Margin = percentile(margins, 0.50);
- cell.bloomSeenFracClean = nSeenObs > 0 ? seenSumClean / nSeenObs : Double.NaN;
- cell.bloomSeenFracMoji = nSeenObs > 0 ? seenSumMoji / nSeenObs : Double.NaN;
- return cell;
- }
-
- private static final class EvalCell {
- int n;
- double meanClean, stdClean;
- double meanMoji;
- double meanMargin, stdMargin;
- double p5Margin, p50Margin;
- double bloomSeenFracClean, bloomSeenFracMoji;
- }
-
- // -----------------------------------------------------------------------
- // FNV-1a hashing for codepoint bigram / unigram + Bloom filter
- // -----------------------------------------------------------------------
-
- private static final long FNV_OFFSET = 0xcbf29ce484222325L;
- private static final long FNV_PRIME = 0x100000001b3L;
-
- private static long fnv1aBigram(int cp1, int cp2) {
- long h = FNV_OFFSET;
- h = (h ^ ((cp1 >>> 24) & 0xFF)) * FNV_PRIME;
- h = (h ^ ((cp1 >>> 16) & 0xFF)) * FNV_PRIME;
- h = (h ^ ((cp1 >>> 8) & 0xFF)) * FNV_PRIME;
- h = (h ^ (cp1 & 0xFF)) * FNV_PRIME;
- h = (h ^ 0xFF) * FNV_PRIME; // separator
- h = (h ^ ((cp2 >>> 24) & 0xFF)) * FNV_PRIME;
- h = (h ^ ((cp2 >>> 16) & 0xFF)) * FNV_PRIME;
- h = (h ^ ((cp2 >>> 8) & 0xFF)) * FNV_PRIME;
- h = (h ^ (cp2 & 0xFF)) * FNV_PRIME;
- return h;
- }
-
- private static long fnv1aUnigram(int cp) {
- long h = FNV_OFFSET;
- h = (h ^ ((cp >>> 24) & 0xFF)) * FNV_PRIME;
- h = (h ^ ((cp >>> 16) & 0xFF)) * FNV_PRIME;
- h = (h ^ ((cp >>> 8) & 0xFF)) * FNV_PRIME;
- h = (h ^ (cp & 0xFF)) * FNV_PRIME;
- return h;
- }
-
- private static long secondaryHash(int cp1, int cp2) {
- // Independent secondary hash for Bloom double-hashing. Just shuffle
- // the inputs differently.
- long h = 0xff51afd7ed558ccdL;
- h = (h ^ Integer.reverse(cp1)) * 0xc4ceb9fe1a85ec53L;
- h = (h ^ Integer.reverse(cp2)) * 0xc4ceb9fe1a85ec53L;
- h ^= h >>> 33;
- return h;
- }
-
- private static void bloomAdd(long[] bits, int bitCount, int k, int cp1, int cp2) {
- long h1 = fnv1aBigram(cp1, cp2);
- long h2 = secondaryHash(cp1, cp2);
- for (int i = 0; i < k; i++) {
- long pos = ((h1 + (long) i * h2) & 0x7FFFFFFFFFFFFFFFL) % bitCount;
- bits[(int) (pos >>> 6)] |= 1L << (pos & 63);
- }
- }
-
- private static boolean bloomContains(long[] bits, int bitCount, int k,
- int cp1, int cp2) {
- long h1 = fnv1aBigram(cp1, cp2);
- long h2 = secondaryHash(cp1, cp2);
- for (int i = 0; i < k; i++) {
- long pos = ((h1 + (long) i * h2) & 0x7FFFFFFFFFFFFFFFL) % bitCount;
- if ((bits[(int) (pos >>> 6)] & (1L << (pos & 63))) == 0) return false;
- }
- return true;
- }
-
- private static long packPair(int cp1, int cp2) {
- return ((long) cp1 << 32) | (cp2 & 0xFFFFFFFFL);
- }
-
- // -----------------------------------------------------------------------
- // I/O and decode utilities (copied from EvalJunkOnCharsetDevtest)
- // -----------------------------------------------------------------------
-
- private static List<byte[]> readRecords(Path file, int maxRecords) throws IOException {
- List<byte[]> records = new ArrayList<>();
- try (FileInputStream fis = new FileInputStream(file.toFile());
- GZIPInputStream gis = new GZIPInputStream(fis);
- DataInputStream dis = new DataInputStream(gis)) {
- while (records.size() < maxRecords) {
- int len;
- try {
- len = dis.readUnsignedShort();
- } catch (EOFException eof) {
- break;
- }
- byte[] rec = new byte[len];
- dis.readFully(rec);
- records.add(rec);
- }
- }
- return records;
- }
-
- private static String decode(byte[] bytes, Charset cs) {
- CharsetDecoder dec = cs.newDecoder()
- .onMalformedInput(CodingErrorAction.REPLACE)
- .onUnmappableCharacter(CodingErrorAction.REPLACE);
- try {
- return dec.decode(ByteBuffer.wrap(bytes)).toString();
- } catch (CharacterCodingException e) {
- return new String(bytes, cs);
- }
- }
-
- private static int[] toCodepoints(String s) {
- int[] cps = new int[s.length()];
- int n = 0;
- for (int i = 0; i < s.length(); ) {
- int cp = s.codePointAt(i);
- cps[n++] = cp;
- i += Character.charCount(cp);
- }
- return Arrays.copyOf(cps, n);
- }
-
- // -----------------------------------------------------------------------
- // Stats
- // -----------------------------------------------------------------------
-
- private static double mean(List<Double> xs) {
- double s = 0;
- int n = 0;
- for (double v : xs) {
- if (!Double.isNaN(v)) {
- s += v;
- n++;
- }
- }
- return n == 0 ? Double.NaN : s / n;
- }
-
- private static double std(List<Double> xs, double mu) {
- if (xs.size() < 2) return 0;
- double s = 0;
- int n = 0;
- for (double v : xs) {
- if (!Double.isNaN(v)) {
- s += (v - mu) * (v - mu);
- n++;
- }
- }
- return n < 2 ? 0 : Math.sqrt(s / (n - 1));
- }
-
- private static double percentile(List<Double> xs, double p) {
- List<Double> sorted = new ArrayList<>(xs);
- sorted.removeIf(v -> Double.isNaN(v));
- if (sorted.isEmpty()) return Double.NaN;
- Collections.sort(sorted);
- int idx = (int) Math.floor(p * (sorted.size() - 1));
- return sorted.get(idx);
- }
-
- // -----------------------------------------------------------------------
- // Model
- // -----------------------------------------------------------------------
-
- private static final class Model {
- final int bigramBuckets;
- final int unigramBuckets;
- final float[] bigramLogP;
- final float[] unigramLogP;
- final long[] bloomBits;
- final int bloomBitCount;
- final int bloomK;
- final double backoffAlpha;
- Model(int bb, int ub, float[] blp, float[] ulp,
- long[] bloom, int bbc, int bk, double a) {
- this.bigramBuckets = bb;
- this.unigramBuckets = ub;
- this.bigramLogP = blp;
- this.unigramLogP = ulp;
- this.bloomBits = bloom;
- this.bloomBitCount = bbc;
- this.bloomK = bk;
- this.backoffAlpha = a;
- }
- }
-}
diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java
index cf52a9eedf..a764444e59 100644
--- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java
+++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java
@@ -20,124 +20,52 @@ import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
-import java.nio.charset.UnsupportedCharsetException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
-import java.util.Set;
import java.util.TreeMap;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
+import org.apache.tika.ml.junkdetect.BigramTables;
+import org.apache.tika.ml.junkdetect.HtmlContentCleaner;
import org.apache.tika.ml.junkdetect.JunkDetector;
-import org.apache.tika.ml.junkdetect.V7Tables;
/**
* Trains the junk detector model from per-script corpus files produced by
* {@link BuildJunkTrainingData}.
*
- * <p>For each script group (identified by a {@code {script}.train.gz} file),
- * four features are trained and then combined by a per-script logistic
- * regression classifier:
- * <ol>
- * <li><b>Byte-bigram log-probability</b>: 256×256 table of log P(b|a) over
- * consecutive byte pairs in the UTF-8 encoding.</li>
- * <li><b>Unicode named-block transition log-probability</b>: N×N table of
- * log P(block_b | block_a), where block ID is determined by
- * {@link Character.UnicodeBlock#of(int)} — one of the ~327 named Unicode
- * blocks plus one extra bucket for unassigned codepoints.</li>
- * <li><b>Control-byte fraction</b>: fraction of bytes in control-character
- * ranges ([0x01–0x08, 0x0B, 0x0C, 0x0E–0x1F, 0x7F]). Stored as
- * {@code −fraction} so the z-score convention matches the other features
- * (higher = cleaner).</li>
- * <li><b>Script-transition log-probability</b>: global table of log P(script_b | script_a)
- * over raw {@link Character.UnicodeScript} values (excluding COMMON, INHERITED, UNKNOWN),
- * pooled across all training scripts (z4).</li>
- * </ol>
+ * <p>z1 (codepoint-bigram log-probability) is trained per script by bucketing
+ * every bigram to its script ({@link JunkDetector#forEachScriptBigram}) and
+ * building a per-script open-addressing bigram table with unigram backoff.
+ * z2 (Unicode block-transition), z3 (control-byte fraction), and z4
+ * (script-transition) are single global document-level features. All features
+ * are calibrated (mu/sigma) and combined by a single global contrastive
+ * combiner producing {@code logit = w1*z1 + ... + w9*z9 + bias}; positive
+ * values indicate clean text.
*
- * <p>All four features are calibrated (mu/sigma) on the dev split so their
- * z-scores are on a common scale. A per-script binary logistic regression
- * classifier is then fit on (z1, z2, z3, z4) using clean dev windows and corrupted
- * versions (inject@5%, char-shuffle) as training examples. The learned weights
- * replace the fixed equal-weight average, allowing the model to automatically
- * downweight noisy features (e.g. high-variance block transitions for MYANMAR)
- * and upweight informative ones (e.g. control-byte fraction for inject@0.01).
- *
- * <p>At inference, the final score is the linear combination
- * {@code w1*z1 + w2*z2 + w3*z3 + w4*z4 + bias}; positive values indicate clean text.
- * The natural threshold is 0 (probability 0.5); use a negative threshold for
- * more conservative junk detection.
- *
- * <p>Output format: {@code JUNKDET1} gzipped binary, <b>version 5</b>.
- * Version 1–4 files can still be loaded by {@code JunkDetector} on the JVM they were trained on.
- *
- * <pre>
- * [8 bytes] magic "JUNKDET1" (ASCII)
- * [1 byte] version = 4
- * [4 bytes] num_scripts (big-endian int)
- * [2 bytes] block_N — number of distinct named Unicode blocks + 1 (unassigned)
- * // Block names section (version 5+): block_N-1 entries for JVM-independence
- * for i in [0, block_N-1):
- * [2 bytes] name length (big-endian ushort)
- * [name bytes] Unicode block name (Character.UnicodeBlock.toString())
- * // Global script-transition section (version 4+)
- * [1 byte] num_script_buckets
- * for each bucket:
- * [2 bytes] name length (big-endian ushort)
- * [name bytes] bucket name (UnicodeScript.name() or "OTHER")
- * [num_script_buckets² × 4 bytes] script-transition log-prob table
- * [4 bytes] mu4 (float32 big-endian)
- * [4 bytes] sigma4 (float32 big-endian)
- * // Per-script data (same as v3 but num_features = 4)
- * for each script (sorted by name):
- * [2 bytes] name length (big-endian ushort)
- * [name bytes] script name (UTF-8)
- * // Feature 1 — byte bigrams
- * [4 bytes] mu1 (float32 big-endian)
- * [4 bytes] sigma1 (float32 big-endian)
- * [65536×4 bytes] byte-bigram log-prob table (256×256)
- * // Feature 2 — block transitions
- * [4 bytes] mu2 (float32 big-endian)
- * [4 bytes] sigma2 (float32 big-endian)
- * [block_N²×4 bytes] block-transition log-prob table
- * // Feature 3 — control-byte fraction
- * [4 bytes] mu3 (float32 big-endian)
- * [4 bytes] sigma3 (float32 big-endian)
- * // Linear classifier weights
- * [1 byte] num_features (= 4 for v4)
- * [4 bytes] w1 (float32 big-endian)
- * [4 bytes] w2 (float32 big-endian)
- * [4 bytes] w3 (float32 big-endian)
- * [4 bytes] w4 (float32 big-endian)
- * [4 bytes] bias (float32 big-endian)
- * </pre>
+ * <p>The output model is written in the {@code JUNKDET1} gzipped binary format
+ * read by {@link JunkDetector#load} — see that method's javadoc for the byte
+ * layout. The file carries {@link JunkDetector#VERSION}; the loader rejects
+ * any other version.
*/
public class TrainJunkModel {
static final String MAGIC = "JUNKDET1";
- /** Sole supported file-format version. Matches JunkDetector.VERSION. */
- static final byte VERSION = 7;
- // -----------------------------------------------------------------------
- // v7 model constants (per-script open-addressing codepoint-bigram tables)
- // -----------------------------------------------------------------------
-
- /** Unigram backoff multiplier. α=1.0 = plain independence; prototype validated. */
- static final float V7_BACKOFF_ALPHA = 1.0f;
+ /** Unigram backoff multiplier. α=1.0 = plain independence. */
+ static final float BACKOFF_ALPHA = 1.0f;
/** Additive smoothing constant for log-prob computation. */
- static final double V7_ADD_ALPHA = 0.01;
+ static final double ADD_ALPHA = 0.01;
/** Number of clean (and corrupted) windows used to train the per-script classifier. */
static final int NUM_CLASSIFIER_SAMPLES = 500;
@@ -156,25 +84,93 @@ public class TrainJunkModel {
static final float CONTROL_BYTE_MIN_SIGMA = 0.005f;
/**
- * Codec pairs used to build wrong-codec remap tables for training.
- * Each entry is {sourceCodec, wrongCodec}: text encoded in sourceCodec but
- * decoded as wrongCodec. Pairs within the same script family (e.g. CP1250↔CP1252)
- * produce wrong-accent distortions that shift characters between Unicode blocks
- * while staying in LATIN. Cross-script pairs (CP1252↔CP1255) additionally change
- * the Unicode script, which z4 also detects.
+ * Full-text byte-level mojibake pairs used by {@link #byteLevelMojibake}.
+ * Each entry is {sourceCodec, wrongCodec}: training text gets encoded in
+ * sourceCodec, then the resulting bytes are re-decoded as wrongCodec to
+ * produce realistic mojibake. Covers SBCS sibling confusion (1252↔1250,
+ * etc.), UTF-8 ↔ Latin (TIKA-4683), and CJK siblings (the GB18030↔EUC-JP
+ * cohort that was -14817 in the 29K eval). For codec pairs that share
+ * an ASCII subset, ASCII-only training samples pass through unchanged
+ * (no-op corruption), so the list is safe to apply across all scripts.
*/
- static final String[][] WRONG_CODEC_PAIRS = {
- {"windows-1252", "windows-1250"}, // Western ↔ Central European (wrong accents)
- {"windows-1250", "windows-1252"}, // reverse
- {"windows-1252", "windows-1257"}, // Western ↔ Baltic (wrong accents)
- {"windows-1257", "windows-1252"}, // reverse
- {"windows-1252", "windows-1254"}, // Western ↔ Turkish (wrong accents)
- {"windows-1251", "windows-1252"}, // Cyrillic → Latin (cross-script)
- {"windows-1252", "windows-1251"}, // Latin → Cyrillic (cross-script)
- {"windows-1253", "windows-1252"}, // Greek → Latin (cross-script)
- {"windows-1252", "windows-1253"}, // Latin → Greek (cross-script)
- {"windows-1255", "windows-1252"}, // Hebrew → Latin (cross-script)
- {"windows-1252", "windows-1255"}, // Latin → Hebrew (the German vcard case)
+ static final String[][] BYTE_LEVEL_MOJIBAKE_PAIRS = {
+ // SBCS Western family
+ {"windows-1252", "windows-1250"},
+ {"windows-1250", "windows-1252"},
+ {"windows-1252", "windows-1257"},
+ {"windows-1257", "windows-1252"},
+ {"windows-1252", "windows-1254"},
+ {"ISO-8859-1", "windows-1252"},
+ {"windows-1252", "ISO-8859-1"},
+ {"x-MacRoman", "windows-1252"},
+ // The exact win-1252/ISO-8859-2 sibling pathology: a win-1252 page with
+ // ©/®/£ symbols read as ISO-8859-2 yields isolated Latin-Extended-A
+ // letters (Š/Ž/Ł). Included as classifier negatives so the LR trains
+ // against this pattern directly.
+ {"windows-1252", "ISO-8859-2"},
+ {"ISO-8859-2", "windows-1252"},
+ // SBCS Cyrillic / Greek / RTL
+ {"windows-1251", "windows-1252"},
+ {"windows-1252", "windows-1251"},
+ {"windows-1253", "windows-1252"},
+ {"windows-1252", "windows-1253"},
+ {"windows-1255", "windows-1252"},
+ {"windows-1252", "windows-1255"},
+ {"windows-1256", "windows-1252"},
+ // Polish ¶ emblem and Central European
+ {"ISO-8859-2", "windows-1250"},
+ {"windows-1250", "ISO-8859-2"},
+ {"ISO-8859-3", "windows-1250"},
+ // Vietnamese
+ {"windows-1258", "windows-1252"},
+ {"windows-1252", "windows-1258"},
+ // UTF-8 → Latin (TIKA-4683 / AIT5)
+ {"UTF-8", "windows-1252"},
+ {"UTF-8", "ISO-8859-1"},
+ // UTF-16 → various — bytes-as-UTF-16 produces dense CJK ideographs
+ // (the AIT5 / TIKA-4683 shape); included for HAN-classifier training
+ // against this cohort.
+ {"UTF-8", "UTF-16LE"},
+ {"UTF-8", "UTF-16BE"},
+ // CJK siblings
+ {"GB18030", "EUC-JP"},
+ {"EUC-JP", "GB18030"}, // reverse
+ {"GB18030", "Shift_JIS"}, // CJK siblings
+ {"Shift_JIS", "GB18030"}, // reverse
+ {"Big5-HKSCS", "GB18030"}, // CJK siblings
+ {"GB18030", "Big5-HKSCS"}, // reverse
+ // Latin → CJK: the SPECIFIC pattern that produces our 66 wrong-CJK
+ // over-adoption cases. Western European accents (0xC0-0xFE in
+ // windows-1252) are valid 2-byte CJK lead bytes; GB18030/Shift_JIS/etc
+ // decoders consume them as the lead of a multi-byte sequence, which
+ // (a) inserts singleton Han characters scattered through Latin text
+ // and (b) eats the byte after each accent. Produces the
+ // long-Latin-with-singleton-HAN fragmentation that z9 measures.
+ // Without these pairs the LATIN classifier never sees this pattern
+ // in its negatives and the LR fits w9 = 0.
+ {"windows-1252", "GB18030"},
+ {"windows-1252", "Shift_JIS"},
+ {"windows-1252", "EUC-JP"},
+ {"windows-1252", "Big5-HKSCS"},
+ {"ISO-8859-1", "GB18030"},
+ {"ISO-8859-1", "Shift_JIS"},
+ };
+
+ /**
+ * The LATIN→CJK subset of {@link #BYTE_LEVEL_MOJIBAKE_PAIRS}, used by
+ * {@link #trainGlobalCombiner} to add extra LATIN→CJK contrastive
+ * negatives. Western-European accents are valid CJK lead bytes, so
+ * decoding Latin text as GB18030/Shift_JIS/etc. scatters singleton Han
+ * characters through it — the high-alternation pattern z9 measures.
+ * Without these the combiner never sees the pattern and fits w9 ≈ 0.
+ */
+ static final String[][] LATIN_TO_CJK_PAIRS = {
+ {"windows-1252", "GB18030"},
+ {"windows-1252", "Shift_JIS"},
+ {"windows-1252", "EUC-JP"},
+ {"windows-1252", "Big5-HKSCS"},
+ {"ISO-8859-1", "GB18030"},
+ {"ISO-8859-1", "Shift_JIS"},
};
/**
@@ -188,9 +184,14 @@ public class TrainJunkModel {
static final int CALIB_SAMPLES = 5000;
public static void main(String[] args) throws IOException {
+ // Training data and output paths are intentionally NOT CLI-configurable.
+ // Single bundled model in git; never train to /tmp/; never maintain
+ // parallel A/B variants. Edit and commit if these need to change.
Path dataDir = Paths.get(System.getProperty("user.home"),
- "datasets", "madlad", "junkdetect");
- Path output = dataDir.resolve("junkdetect.bin");
+ "data", "junk-augmented-symbolboost");
+ Path output = Paths.get(
+ "tika-ml/tika-ml-junkdetect/src/main/resources",
+ "org/apache/tika/ml/junkdetect/junkdetect.bin");
// Durable training parameters live in JunkDetectorTrainingConfig; this
// tool deliberately refuses CLI overrides so a built model file's
@@ -211,32 +212,19 @@ public class TrainJunkModel {
System.exit(1);
}
- for (int i = 0; i < args.length; i++) {
- switch (args[i]) {
- case "--data-dir":
- dataDir = Paths.get(args[++i]);
- break;
- case "--output":
- output = Paths.get(args[++i]);
- break;
- case "--bloom-bits":
- case "--min-bigram-count":
- System.err.println("ERROR: " + args[i] + " is no longer a CLI option."
- + " Edit JunkDetectorTrainingConfig and commit the change instead.");
- System.exit(1);
- break;
- default:
- System.err.println("Unknown argument: " + args[i]);
- printUsage();
- System.exit(1);
- }
+ if (args.length > 0) {
+ System.err.println("TrainJunkModel takes no arguments. Data-dir and"
+ + " output path are baked in; edit the source if they need"
+ + " to change. Other training parameters live in"
+ + " JunkDetectorTrainingConfig.");
+ System.exit(1);
}
System.out.println("=== TrainJunkModel ===");
System.out.println(" data-dir: " + dataDir);
System.out.println(" output: " + output);
- System.out.println(" --- v7 format constants (TrainJunkModel) ---");
- System.out.printf( " backoff_alpha: %.2f%n", V7_BACKOFF_ALPHA);
+ System.out.println(" --- format constants (TrainJunkModel) ---");
+ System.out.printf( " backoff_alpha: %.2f%n", BACKOFF_ALPHA);
System.out.println(" --- config (JunkDetectorTrainingConfig) ---");
System.out.printf( " min_bigram_count: %d%n", minBigramCount);
System.out.printf( " oa_load_factor: %.2f%n", loadFactor);
@@ -254,10 +242,6 @@ public class TrainJunkModel {
long t0 = System.currentTimeMillis();
TreeMap<String, float[]> f1Calibrations = new TreeMap<>();
- TreeMap<String, float[]> blockTables = new TreeMap<>();
- TreeMap<String, float[]> blockCalibrations = new TreeMap<>();
- TreeMap<String, float[]> controlCalibrations = new TreeMap<>();
- TreeMap<String, float[]> classifierWeights = new TreeMap<>();
TreeMap<String, Path> trainFilePaths = new TreeMap<>();
List<Path> allTrainFiles = new ArrayList<>();
@@ -275,129 +259,115 @@ public class TrainJunkModel {
}
// -----------------------------------------------------------------------
- // Phase 1 — per-script F1 tables (V7), F1 calibration, F2 block tables,
- // F3 control-byte calibration
+ // Phase 1 — bucket-by-script bigram tables (per real script + COMMON).
+ // ONE tally pass over every train file via forEachScriptBigram: digits
+ // skipped, COMMON glue folded into the adjacent script, both-COMMON
+ // bigrams charged to the COMMON bucket. No runs, no sentinels. COMMON
+ // is a normal bucket (its own table + calibration + combiner weight).
// -----------------------------------------------------------------------
- TreeMap<String, V7Tables> f1TablesByScript = new TreeMap<>();
- System.out.println("\n--- Phase 1: per-script F1 tables + calibrations ---");
+ System.out.println("\n--- Phase 1: bucket-by-script F1 tables ---");
+ Map<String, HashMap<Long, long[]>> pairsByScript = new HashMap<>();
+ Map<String, HashMap<Integer, long[]>> unigramsByScript = new HashMap<>();
+ Map<String, long[]> totalsByScript = new HashMap<>();
for (Path trainFile : trainFiles) {
+ allTrainFiles.add(trainFile);
String filename = trainFile.getFileName().toString();
String script = filename.substring(0, filename.length() - ".train.gz".length())
.toUpperCase();
+ trainFilePaths.put(script, trainFile);
+ tallyFileBuckets(trainFile, pairsByScript, unigramsByScript, totalsByScript);
+ }
- System.out.printf("%n [%s]%n", script);
- allTrainFiles.add(trainFile);
-
- t0 = System.currentTimeMillis();
- System.out.print(" Training V7 F1 tables (cp index + OA)..");
- V7Tables v7 = trainV7TablesForScript(trainFile, minBigramCount,
- loadFactor, keyIndexBits);
- System.out.printf(" done (%dms)%n", System.currentTimeMillis() - t0);
- System.out.println(v7.statsString());
- f1TablesByScript.put(script, v7);
-
- t0 = System.currentTimeMillis();
- System.out.print(" Training named-block table... ");
- float[] blockTable = trainBlockTable(trainFile);
- System.out.printf("done (%dms)%n", System.currentTimeMillis() - t0);
-
- t0 = System.currentTimeMillis();
- System.out.print(" Calibrating F1 (cp-hash) on train.. ");
- float[] f1Cal = calibrateF1PerScript(trainFile, v7);
- System.out.printf("done — mu=%.4f sigma=%.4f (%dms)%n",
- f1Cal[0], f1Cal[1], System.currentTimeMillis() - t0);
-
- t0 = System.currentTimeMillis();
- System.out.print(" Calibrating named blocks on train...");
- float[] blockCal = computeBlockCalibration(trainFile, blockTable);
- System.out.printf("done — mu=%.4f sigma=%.4f (%dms)%n",
- blockCal[0], blockCal[1], System.currentTimeMillis() - t0);
-
+ TreeMap<String, BigramTables> f1TablesByScript = new TreeMap<>();
+ for (String script : new java.util.TreeSet<>(pairsByScript.keySet())) {
t0 = System.currentTimeMillis();
- System.out.print(" Calibrating control bytes on train..");
- float[] controlCal = computeControlByteCalibration(trainFile);
- System.out.printf("done — mu=%.6f sigma=%.6f (%dms)%n",
- controlCal[0], controlCal[1], System.currentTimeMillis() - t0);
-
- trainFilePaths.put(script, trainFile);
+ long[] totals = totalsByScript.get(script);
+ BigramTables tables = buildBigramTablesFromCounts(pairsByScript.get(script),
+ unigramsByScript.get(script), totals[0],
+ minBigramCount, loadFactor, keyIndexBits);
+ f1TablesByScript.put(script, tables);
+ System.out.printf(" [%s] %s (%dms)%n", script, tables.statsString(),
+ System.currentTimeMillis() - t0);
+ }
- f1Calibrations.put(script, f1Cal);
- blockTables.put(script, blockTable);
- blockCalibrations.put(script, blockCal);
- controlCalibrations.put(script, controlCal);
- // Placeholder — set in Phase 3
- classifierWeights.put(script, new float[]{1f / 4, 1f / 4, 1f / 4, 1f / 4, 0f});
+ // Phase 1b — per-script (incl COMMON) z1 calibration on the bucket mean.
+ // Sample windows from all files ONCE; for each window distribute every
+ // present script bucket's mean to that script's score list, then muSigma.
+ // This is exactly the inference z1 input, so calibration cannot drift.
+ System.out.println("\n--- Phase 1b: per-script z1 calibration ---");
+ Map<String, List<Double>> z1ScoresByScript = new HashMap<>();
+ int calibSeed = 42;
+ for (Path f : trainFiles) {
+ for (String window : sampleSubstrings(f, CALIB_SAMPLES, CALIB_LENGTHS, calibSeed++)) {
+ int[] cps = window.codePoints().toArray();
+ Map<String, double[]> b = JunkDetector.bucketSumsAndCounts(cps, f1TablesByScript);
+ for (Map.Entry<String, double[]> e : b.entrySet()) {
+ if (e.getValue()[1] > 0) {
+ z1ScoresByScript.computeIfAbsent(e.getKey(), k -> new ArrayList<>())
+ .add(e.getValue()[0] / e.getValue()[1]);
+ }
+ }
+ }
+ }
+ for (String script : f1TablesByScript.keySet()) {
+ List<Double> scores = z1ScoresByScript.getOrDefault(script, new ArrayList<>());
+ float[] cal = scores.isEmpty() ? new float[]{0f, 1f} : muSigma(scores);
+ f1Calibrations.put(script, cal);
+ System.out.printf(" [%s] mu=%.4f sigma=%.4f (%,d windows)%n",
+ script, cal[0], cal[1], scores.size());
}
// -----------------------------------------------------------------------
- // Phase 2 — global script-transition table + supporting pools
+ // Phase 2 — global script-transition table (z4) + single GLOBAL block
+ // table (z2) + single GLOBAL control-byte calibration (z3).
// -----------------------------------------------------------------------
- System.out.println("\n--- Phase 2: global script-transition table ---");
+ System.out.println("\n--- Phase 2: global script-transition + block + control ---");
List<String> scriptBuckets = buildScriptBuckets();
int numScriptBuckets = scriptBuckets.size();
Map<String, Integer> scriptBucketMap = new LinkedHashMap<>();
for (int i = 0; i < numScriptBuckets; i++) {
scriptBucketMap.put(scriptBuckets.get(i), i);
}
- System.out.printf(" %d script buckets (including OTHER)%n", numScriptBuckets);
-
- t0 = System.currentTimeMillis();
- System.out.print(" Training script-transition table... ");
float[] scriptTransTable = trainScriptTransitionTable(allTrainFiles, scriptBucketMap, numScriptBuckets);
- System.out.printf("done (%dms)%n", System.currentTimeMillis() - t0);
-
- t0 = System.currentTimeMillis();
- System.out.print(" Calibrating script transitions... ");
+ scriptTransTable = quantizeDequantizeRoundTrip(scriptTransTable);
float[] scriptTransCal = calibrateScriptTransitions(allTrainFiles, scriptTransTable,
scriptBucketMap, numScriptBuckets);
- System.out.printf("done — mu=%.4f sigma=%.4f (%dms)%n",
- scriptTransCal[0], scriptTransCal[1], System.currentTimeMillis() - t0);
+ System.out.printf(" scriptTrans: mu=%.4f sigma=%.4f%n",
+ scriptTransCal[0], scriptTransCal[1]);
- t0 = System.currentTimeMillis();
- System.out.print(" Collecting per-script codepoint pools... ");
- Map<String, List<Integer>> scriptCodepoints = collectScriptCodepoints(allTrainFiles, 200);
- System.out.printf("done — %d scripts (%dms)%n",
- scriptCodepoints.size(), System.currentTimeMillis() - t0);
+ float[] blockTable = quantizeDequantizeRoundTrip(trainGlobalBlockTable(allTrainFiles));
+ float[] blockCal = computeGlobalBlockCalibration(allTrainFiles, blockTable);
+ System.out.printf(" block: mu=%.4f sigma=%.4f%n", blockCal[0], blockCal[1]);
- System.out.print(" Building wrong-codec remap tables... ");
- List<Map<Character, Character>> remapTables = new ArrayList<>();
- for (String[] pair : WRONG_CODEC_PAIRS) {
- Map<Character, Character> table = buildRemapTable(pair[0], pair[1]);
- if (!table.isEmpty()) remapTables.add(table);
- }
- System.out.printf("%d tables built%n", remapTables.size());
+ float[] controlCal = computeGlobalControlCalibration(allTrainFiles);
+ System.out.printf(" control: mu=%.6f sigma=%.6f%n", controlCal[0], controlCal[1]);
// -----------------------------------------------------------------------
- // Phase 3 — per-script linear classifiers using v6 features
+ // Phase 3 — ONE global combiner over z1..z9, trained pointwise
+ // (clean>garbage) + contrastive (correct-decode>wrong, incl. RTL
+ // logical>reversed). Features extracted through a temp model so they are
+ // exactly the inference features. Applies to every bucket incl COMMON.
// -----------------------------------------------------------------------
- System.out.println("\n--- Phase 3: per-script linear classifiers (z1,z2,z3,z4) ---");
- for (String script : f1Calibrations.keySet()) {
- Path trainFile = trainFilePaths.get(script);
- if (trainFile == null) {
- System.out.printf(" [%s] WARNING: no train file, keeping equal-weight defaults%n", script);
- continue;
- }
- t0 = System.currentTimeMillis();
- System.out.printf(" [%s] training classifier... ", script);
- float[] weights = trainClassifierV7(trainFile,
- f1TablesByScript.get(script), f1Calibrations.get(script),
- blockTables.get(script), blockCalibrations.get(script),
- controlCalibrations.get(script),
- scriptTransTable, scriptTransCal, scriptBucketMap, numScriptBuckets,
- scriptCodepoints, remapTables);
- classifierWeights.put(script, weights);
- System.out.printf("done — w=[%.3f,%.3f,%.3f,%.3f] bias=%.3f (%dms)%n",
- weights[0], weights[1], weights[2], weights[3], weights[4],
- System.currentTimeMillis() - t0);
- }
+ System.out.println("\n--- Phase 3: global contrastive combiner ---");
+ float[] placeholder = new float[]{1f, 0f, 0f, 0f, 0f, 0f, 0f, 0f, 0f, 0f};
+ Path tmpModel = Files.createTempFile("junkdetect-feat", ".bin");
+ saveModel(f1TablesByScript, f1Calibrations, blockTable, blockCal, controlCal,
+ placeholder, scriptBuckets, scriptTransTable, scriptTransCal, tmpModel);
+ JunkDetector featExtractor = JunkDetector.loadFromPath(tmpModel);
+ Files.deleteIfExists(tmpModel);
+
+ t0 = System.currentTimeMillis();
+ float[] combiner = trainGlobalCombiner(featExtractor, trainFilePaths);
+ System.out.printf(
+ " global w=[%.3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f] bias=%.3f (%dms)%n",
+ combiner[0], combiner[1], combiner[2], combiner[3], combiner[4], combiner[5],
+ combiner[6], combiner[7], combiner[8], combiner[9],
+ System.currentTimeMillis() - t0);
System.out.printf("%nWriting model (%d scripts, blockN=%d, scriptBuckets=%d) → %s%n",
- f1Calibrations.size(), blockN, numScriptBuckets, output);
- saveModelV7(f1TablesByScript, f1Calibrations,
- blockTables, blockCalibrations,
- controlCalibrations, classifierWeights,
- scriptBuckets, scriptTransTable, scriptTransCal,
- output);
+ f1TablesByScript.size(), blockN, numScriptBuckets, output);
+ saveModel(f1TablesByScript, f1Calibrations, blockTable, blockCal, controlCal,
+ combiner, scriptBuckets, scriptTransTable, scriptTransCal, output);
System.out.printf("Model size: %,d bytes (%.1f KB)%n",
Files.size(output), Files.size(output) / 1024.0);
System.out.println("Done.");
@@ -407,41 +377,6 @@ public class TrainJunkModel {
// Training
// -----------------------------------------------------------------------
- /**
- * Trains a {@code N × N} block-transition log-probability table where
- * {@code N = UnicodeBlockRanges.bucketCount()}. Block bucketing uses
- * the JVM-independent {@link UnicodeBlockRanges} table.
- *
- * @return float[N*N] where index {@code a*N+b} = log P(block_b | block_a)
- */
- static float[] trainBlockTable(Path trainGz) throws IOException {
... 4328 lines suppressed ...