You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by rz...@apache.org on 2026/09/18 13:07:15 UTC
(opennlp) 01/02: OPENNLP-1946: Add BoundedUrlCharSequenceNormalizer for complete URLs and document the limits of the default
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch OPENNLP-1946-url-normalizer
in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit a40a42ad1994d7c7afe9c29295a350a1f360858f
Author: Kristian Rickert <kr...@apache.org>
AuthorDate: Tue Sep 15 22:29:14 2026 -0400
OPENNLP-1946: Add BoundedUrlCharSequenceNormalizer for complete URLs and document the limits of the default
The default normalizer and the language detector chain are unchanged, and the
class Javadoc states the limits found with the WHATWG URL test data. The new
class removes http and https URLs with ports, userinfo, percent escapes, IPv6
hosts and non-ASCII paths, bounded by whitespace and delimiters, with a left
boundary before the scheme, in linear time. The manual describes both and how
to opt in.
---
.../BoundedUrlCharSequenceNormalizer.java | 271 ++++++++++++++++++++
.../tools/util/normalizer/MailAddressScan.java | 112 ++++++++
.../util/normalizer/UrlCharSequenceNormalizer.java | 100 ++------
opennlp-docs/src/docbkx/langdetect.xml | 30 ++-
.../BoundedUrlCharSequenceNormalizerTest.java | 283 +++++++++++++++++++++
.../normalizer/UrlCharSequenceNormalizerTest.java | 41 +++
6 files changed, 755 insertions(+), 82 deletions(-)
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/BoundedUrlCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/BoundedUrlCharSequenceNormalizer.java
new file mode 100644
index 000000000..d27f36402
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/BoundedUrlCharSequenceNormalizer.java
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util.normalizer;
+
+import opennlp.tools.util.StringUtil;
+
+/**
+ * A {@link CharSequenceNormalizer} that replaces each {@code http} or {@code https} URL and
+ * each email address with one space, taking a URL as a whole the way it is bounded in
+ * running text.
+ *
+ * <p>A URL is a scheme, {@code ://}, and a body. The scheme is {@code http} or {@code https}
+ * in any letter case and starts the text or follows a code point that is neither a letter,
+ * a digit, nor one of {@code _ + - . :}, so the {@code https} inside {@code git+https://}
+ * or {@code blob:https://} is not a URL of its own. The body runs to the end of the text or
+ * to the first Unicode whitespace, control character, unpaired surrogate, typographic
+ * quotation mark, or one of {@code < > "}; it therefore includes a port, userinfo, percent
+ * escapes, a bracketed IPv6 host, and non-ASCII host names and paths. Trailing sentence punctuation
+ * ({@code . , ; : ! ?}) and a closing bracket without its opening bracket inside the body
+ * are not part of the URL. A URL with another scheme is kept as it is, and no email address
+ * is matched inside it. This differs from {@link UrlCharSequenceNormalizer}, the normalizer
+ * existing language detector models were trained with; a model trained with one normalizer
+ * must be decoded with the same one.
+ *
+ * <p>Email addresses are matched as in {@link UrlCharSequenceNormalizer}.
+ *
+ * @see <a href="https://url.spec.whatwg.org/">WHATWG URL Standard</a>
+ * @since 3.0.0
+ */
+public final class BoundedUrlCharSequenceNormalizer implements CharSequenceNormalizer {
+
+ private static final long serialVersionUID = -1024195904855005L;
+
+ private static final String SCHEME_SEPARATOR = "://";
+ private static final String HTTP = "http";
+ private static final String HTTPS = "https";
+
+ /** The characters a scheme continues with after its first letter, per RFC 3986. */
+ private static final CodePointSet SCHEME_BODY =
+ MailAddressScan.ASCII_ALNUM.union(CodePointSet.of('+', '-', '.'));
+
+ /** Code points that, before the scheme, make it part of a larger token. */
+ private static final CodePointSet SCHEME_NEIGHBORS = CodePointSet.of('_', '+', '-', '.', ':');
+
+ /** Delimiters that end a URL body besides whitespace and control characters. */
+ private static final CodePointSet BODY_DELIMITERS = CodePointSet.of('<', '>', '"');
+
+ /** Sentence punctuation given back when it trails a URL. */
+ private static final CodePointSet TRAILING_PUNCTUATION =
+ CodePointSet.of('.', ',', ';', ':', '!', '?');
+
+ private static final BoundedUrlCharSequenceNormalizer INSTANCE =
+ new BoundedUrlCharSequenceNormalizer();
+
+ private BoundedUrlCharSequenceNormalizer() {
+ }
+
+ /** {@return the shared, stateless instance} */
+ public static BoundedUrlCharSequenceNormalizer getInstance() {
+ return INSTANCE;
+ }
+
+ /**
+ * {@inheritDoc}
+ * URLs and email addresses are found in one pass, so an address inside a URL of any
+ * scheme is never matched on its own.
+ */
+ @Override
+ public CharSequence normalize(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("The text must not be null.");
+ }
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final int bodyStart = schemeEnd(text, i);
+ if (bodyStart != -1) {
+ final int end = bodyEnd(text, bodyStart);
+ if (end > bodyStart) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ if (isHttpScheme(text, i, bodyStart - SCHEME_SEPARATOR.length())) {
+ out.append(' ');
+ } else {
+ out.append(text, i, end);
+ }
+ i = end;
+ continue;
+ }
+ }
+ final int mailEnd = MailAddressScan.matchEnd(text, i);
+ if (mailEnd > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = mailEnd;
+ continue;
+ }
+ final int codePoint = Character.codePointAt(text, i);
+ if (out != null) {
+ out.appendCodePoint(codePoint);
+ }
+ i += Character.charCount(codePoint);
+ }
+ return out == null ? text : out.toString();
+ }
+
+ /**
+ * Finds the end of a scheme and its separator that start at an index.
+ *
+ * @param text The text. Must not be {@code null}.
+ * @param start The index the scheme must start at, inside the text.
+ * @return The index after {@code ://}, or {@code -1} if no scheme with a left boundary
+ * starts there.
+ */
+ private int schemeEnd(CharSequence text, int start) {
+ if (start > 0) {
+ final int before = Character.codePointBefore(text, start);
+ if (Character.isLetterOrDigit(before) || SCHEME_NEIGHBORS.contains(before)) {
+ return -1;
+ }
+ }
+ final int length = text.length();
+ if (!isAsciiLetter(text.charAt(start))) {
+ return -1;
+ }
+ int at = start + 1;
+ while (at < length && SCHEME_BODY.contains(text.charAt(at))) {
+ at++;
+ }
+ final int end = at + SCHEME_SEPARATOR.length();
+ if (end > length) {
+ return -1;
+ }
+ for (int k = 0; k < SCHEME_SEPARATOR.length(); k++) {
+ if (text.charAt(at + k) != SCHEME_SEPARATOR.charAt(k)) {
+ return -1;
+ }
+ }
+ return end;
+ }
+
+ /**
+ * Finds the end of a URL body that starts at an index.
+ *
+ * @param text The text. Must not be {@code null}.
+ * @param bodyStart The index after the scheme separator.
+ * @return The exclusive end of the body, or {@code bodyStart} if the body is empty.
+ */
+ private int bodyEnd(CharSequence text, int bodyStart) {
+ final int length = text.length();
+ int end = bodyStart;
+ while (end < length) {
+ final int codePoint = Character.codePointAt(text, end);
+ if (endsBody(codePoint)) {
+ break;
+ }
+ end += Character.charCount(codePoint);
+ }
+ return trimTrailing(text, bodyStart, end);
+ }
+
+ /**
+ * Tests whether a code point ends a URL body.
+ *
+ * @param codePoint The code point, an unpaired surrogate read as its own code point included.
+ * @return {@code true} for whitespace, a control character, an unpaired surrogate, a
+ * typographic quotation mark, or a delimiter.
+ */
+ private boolean endsBody(int codePoint) {
+ final int type = Character.getType(codePoint);
+ return StringUtil.isUnicodeWhitespace(codePoint)
+ || type == Character.CONTROL
+ || type == Character.INITIAL_QUOTE_PUNCTUATION
+ || type == Character.FINAL_QUOTE_PUNCTUATION
+ || (codePoint <= Character.MAX_VALUE && Character.isSurrogate((char) codePoint))
+ || BODY_DELIMITERS.contains(codePoint);
+ }
+
+ /**
+ * Gives back trailing sentence punctuation and closing brackets that have no opening
+ * bracket inside the body.
+ *
+ * @param text The text. Must not be {@code null}.
+ * @param bodyStart The index the body starts at.
+ * @param end The exclusive end of the body as scanned.
+ * @return The exclusive end of the URL.
+ */
+ private int trimTrailing(CharSequence text, int bodyStart, int end) {
+ int roundBalance = 0;
+ int squareBalance = 0;
+ int curlyBalance = 0;
+ for (int i = bodyStart; i < end; i++) {
+ switch (text.charAt(i)) {
+ case '(' -> roundBalance++;
+ case ')' -> roundBalance--;
+ case '[' -> squareBalance++;
+ case ']' -> squareBalance--;
+ case '{' -> curlyBalance++;
+ case '}' -> curlyBalance--;
+ default -> { }
+ }
+ }
+ while (end > bodyStart) {
+ final char last = text.charAt(end - 1);
+ if (TRAILING_PUNCTUATION.contains(last)) {
+ end--;
+ } else if (last == ')' && roundBalance < 0) {
+ roundBalance++;
+ end--;
+ } else if (last == ']' && squareBalance < 0) {
+ squareBalance++;
+ end--;
+ } else if (last == '}' && curlyBalance < 0) {
+ curlyBalance++;
+ end--;
+ } else {
+ break;
+ }
+ }
+ return end;
+ }
+
+ /**
+ * Tests whether the scheme between two indexes is {@code http} or {@code https} in any
+ * letter case.
+ */
+ private boolean isHttpScheme(CharSequence text, int start, int end) {
+ final int schemeLength = end - start;
+ final String expected;
+ if (schemeLength == HTTP.length()) {
+ expected = HTTP;
+ } else if (schemeLength == HTTPS.length()) {
+ expected = HTTPS;
+ } else {
+ return false;
+ }
+ for (int k = 0; k < schemeLength; k++) {
+ if (Character.toLowerCase(text.charAt(start + k)) != expected.charAt(k)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private boolean isAsciiLetter(char c) {
+ return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
+ }
+
+ private Object readResolve() {
+ return INSTANCE;
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/MailAddressScan.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/MailAddressScan.java
new file mode 100644
index 000000000..a7d4662c6
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/MailAddressScan.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util.normalizer;
+
+/**
+ * Finds email addresses in text for the URL normalizers. An address is a maximal run of the
+ * local-part set {@code [-+_.0-9A-Za-z]} with a left neighbor outside that set, an
+ * {@code @}, and a domain run out of {@code [-.0-9A-Za-z]} that does not start with a dot
+ * and spans at least two chars.
+ */
+final class MailAddressScan {
+
+ /** The ASCII letters and digits. */
+ static final CodePointSet ASCII_ALNUM = CodePointSet.ofRange('0', '9')
+ .union(CodePointSet.ofRange('A', 'Z'))
+ .union(CodePointSet.ofRange('a', 'z'));
+
+ /** The mail local-part set, also the left-neighbor exclusion set: {@code [-+_.0-9A-Za-z]}. */
+ private static final CodePointSet MAIL_LOCAL =
+ ASCII_ALNUM.union(CodePointSet.of('-', '+', '_', '.'));
+
+ /** The set a domain may start with: {@code [-0-9A-Za-z]}. */
+ private static final CodePointSet MAIL_DOMAIN_START = ASCII_ALNUM.union(CodePointSet.of('-'));
+
+ /** The set a domain continues with: {@code [-.0-9A-Za-z]}. */
+ private static final CodePointSet MAIL_DOMAIN = MAIL_DOMAIN_START.union(CodePointSet.of('.'));
+
+ private static final char AT = '@';
+ private static final int MIN_DOMAIN_LENGTH = 2;
+
+ private MailAddressScan() {
+ }
+
+ /**
+ * Replaces each email address with one space.
+ *
+ * @param text The text to scan. Must not be {@code null}.
+ * @return The input itself when no address matched, otherwise the normalized copy.
+ */
+ static CharSequence removeAll(CharSequence text) {
+ final int length = text.length();
+ StringBuilder out = null;
+ int i = 0;
+ while (i < length) {
+ final int end = matchEnd(text, i);
+ if (end > i) {
+ if (out == null) {
+ out = new StringBuilder(length).append(text, 0, i);
+ }
+ out.append(' ');
+ i = end;
+ } else {
+ if (out != null) {
+ out.append(text.charAt(i));
+ }
+ i++;
+ }
+ }
+ return out == null ? text : out.toString();
+ }
+
+ /**
+ * Finds the end of an email address that starts at an index.
+ *
+ * @param text The text to look into. Must not be {@code null}.
+ * @param start The index the address must start at, inside the text.
+ * @return The exclusive end of the address, or {@code -1} if none starts there.
+ */
+ static int matchEnd(CharSequence text, int start) {
+ if (start > 0 && MAIL_LOCAL.contains(text.charAt(start - 1))) {
+ return -1;
+ }
+ if (!MAIL_LOCAL.contains(text.charAt(start))) {
+ return -1;
+ }
+ final int length = text.length();
+ int at = start + 1;
+ while (at < length && MAIL_LOCAL.contains(text.charAt(at))) {
+ at++;
+ }
+ if (at >= length || text.charAt(at) != AT) {
+ return -1;
+ }
+ final int domainStart = at + 1;
+ if (domainStart >= length || !MAIL_DOMAIN_START.contains(text.charAt(domainStart))) {
+ return -1;
+ }
+ int end = domainStart + 1;
+ while (end < length && MAIL_DOMAIN.contains(text.charAt(end))) {
+ end++;
+ }
+ if (end - domainStart < MIN_DOMAIN_LENGTH) {
+ return -1;
+ }
+ return end;
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java
index f843f1109..17f3fb191 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizer.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package opennlp.tools.util.normalizer;
/**
@@ -31,28 +32,29 @@ package opennlp.tools.util.normalizer;
* {@code [-.0-9A-Za-z]} that must not start with a dot and must span at least two
* chars, become one space.</li>
* </ol>
+ *
+ * <p>The output is part of the features that language detector models are trained on, so
+ * the following limits are kept as they are (OPENNLP-1946):</p>
+ * <ul>
+ * <li>A URL with a port, userinfo, a percent escape, a non-ASCII host or path, a bracketed
+ * IPv6 host, or ASCII punctuation outside the body set is removed only up to that
+ * character: {@code http://f:21/} becomes {@code " :21/"}, and {@code http://[::1]/} is
+ * kept.</li>
+ * <li>The scheme needs no left boundary, so {@code git+https://github.com/x} becomes
+ * {@code "git+ "}.</li>
+ * <li>The email pass also matches inside the userinfo of a URL with another scheme:
+ * {@code telnet://user:pass@foobar.com:23/} becomes {@code "telnet://user: :23/"}.</li>
+ * </ul>
+ * {@link BoundedUrlCharSequenceNormalizer} removes whole URLs as they are bounded in running
+ * text and has none of these limits.
*/
public class UrlCharSequenceNormalizer implements CharSequenceNormalizer {
private static final long serialVersionUID = 2023145028634552389L;
- private static final CodePointSet ASCII_ALNUM = CodePointSet.ofRange('0', '9')
- .union(CodePointSet.ofRange('A', 'Z'))
- .union(CodePointSet.ofRange('a', 'z'));
-
/** The URL body set: {@code [-_.?&~;+=/#0-9A-Za-z]}. */
- private static final CodePointSet URL_BODY =
- ASCII_ALNUM.union(CodePointSet.of('-', '_', '.', '?', '&', '~', ';', '+', '=', '/', '#'));
-
- /** The mail local-part set, also the left-neighbor exclusion set: {@code [-+_.0-9A-Za-z]}. */
- private static final CodePointSet MAIL_LOCAL =
- ASCII_ALNUM.union(CodePointSet.of('-', '+', '_', '.'));
-
- /** The set a domain may start with: {@code [-0-9A-Za-z]}. */
- private static final CodePointSet MAIL_DOMAIN_START = ASCII_ALNUM.union(CodePointSet.of('-'));
-
- /** The set a domain continues with: {@code [-.0-9A-Za-z]}. */
- private static final CodePointSet MAIL_DOMAIN = MAIL_DOMAIN_START.union(CodePointSet.of('.'));
+ private static final CodePointSet URL_BODY = MailAddressScan.ASCII_ALNUM
+ .union(CodePointSet.of('-', '_', '.', '?', '&', '~', ';', '+', '=', '/', '#'));
private static final UrlCharSequenceNormalizer INSTANCE = new UrlCharSequenceNormalizer();
@@ -69,7 +71,7 @@ public class UrlCharSequenceNormalizer implements CharSequenceNormalizer {
if (text == null) {
throw new IllegalArgumentException("The text must not be null.");
}
- return removeMailAddresses(removeUrls(text));
+ return MailAddressScan.removeAll(removeUrls(text));
}
/**
@@ -149,68 +151,4 @@ public class UrlCharSequenceNormalizer implements CharSequenceNormalizer {
}
return true;
}
-
- /**
- * Replaces each email address with one space.
- *
- * @param text The text to scan; never null.
- * @return The input itself when nothing matched, otherwise the normalized copy.
- */
- private CharSequence removeMailAddresses(CharSequence text) {
- final int length = text.length();
- StringBuilder out = null;
- int i = 0;
- while (i < length) {
- final int end = matchMailEnd(text, i);
- if (end > i) {
- if (out == null) {
- out = new StringBuilder(length).append(text, 0, i);
- }
- out.append(' ');
- i = end;
- } else {
- if (out != null) {
- out.append(text.charAt(i));
- }
- i++;
- }
- }
- return out == null ? text : out.toString();
- }
-
- /**
- * {@return the exclusive end of an email match starting at {@code start}, or {@code -1} if
- * there is none}
- *
- * @param text The text to look into; never null.
- * @param start The index the match must start at.
- */
- private int matchMailEnd(CharSequence text, int start) {
- if (start > 0 && MAIL_LOCAL.contains(text.charAt(start - 1))) {
- return -1; // a match never starts inside a local-part run
- }
- if (!MAIL_LOCAL.contains(text.charAt(start))) {
- return -1;
- }
- final int length = text.length();
- int at = start + 1;
- while (at < length && MAIL_LOCAL.contains(text.charAt(at))) {
- at++;
- }
- if (at >= length || text.charAt(at) != '@') {
- return -1;
- }
- final int domainStart = at + 1;
- if (domainStart >= length || !MAIL_DOMAIN_START.contains(text.charAt(domainStart))) {
- return -1;
- }
- int end = domainStart + 1;
- while (end < length && MAIL_DOMAIN.contains(text.charAt(end))) {
- end++;
- }
- if (end - domainStart < 2) {
- return -1;
- }
- return end;
- }
}
diff --git a/opennlp-docs/src/docbkx/langdetect.xml b/opennlp-docs/src/docbkx/langdetect.xml
index 7c8e713f1..7298ddb34 100644
--- a/opennlp-docs/src/docbkx/langdetect.xml
+++ b/opennlp-docs/src/docbkx/langdetect.xml
@@ -52,7 +52,13 @@ under the License.
</row>
<row>
<entry>UrlCharSequenceNormalizer</entry>
- <entry>Replaces URLs and E-Mails by a blank space.</entry>
+ <entry>Replaces URLs and E-Mails by a blank space. A URL is removed up to the
+ first character outside its ASCII body set, so a port, a percent escape,
+ or a non-ASCII host cuts the match short; existing models were trained
+ with this output. BoundedUrlCharSequenceNormalizer, not in the default
+ chain, removes each http or https URL as a whole up to the next
+ whitespace or delimiter and leaves URLs with other schemes untouched; a
+ model trained with it must be decoded with it.</entry>
</row>
<row>
<entry>TwitterCharSequenceNormalizer</entry>
@@ -116,6 +122,28 @@ Language[] languages = myCategorizer.predictLanguages(null);]]>
</para>
</section>
+ <section xml:id="tools.langdetect.bounded.urls">
+ <title>Selecting URL normalization</title>
+ <para>
+ For new training pipelines, <code>BoundedUrlCharSequenceNormalizer.getInstance()</code>
+ can be passed to <code>DefaultLanguageDetectorContextGenerator</code> in place of
+ <code>UrlCharSequenceNormalizer</code>. It removes HTTP and HTTPS URLs including
+ ports, user information, percent escapes, bracketed IPv6 hosts, and non-ASCII paths.
+ It keeps trailing sentence punctuation and unmatched closing brackets. A scheme
+ must start the text or follow a character other than a letter, digit, underscore,
+ plus, hyphen, period, or colon; for example, <code>see:http://example.org</code>
+ is not recognized as an HTTP URL. Other schemes, such as <code>git+https</code>,
+ are kept, without removing email-like text inside their URL bodies.
+ </para>
+ <para>
+ This normalizer identifies URL boundaries in running text; it does not validate
+ URLs against the full WHATWG standard. The default factory continues to use
+ <code>UrlCharSequenceNormalizer</code>. To opt in, supply a custom
+ <code>LanguageDetectorFactory</code> with a <code>getContextGenerator()</code>
+ that uses the new normalizer, and use that factory for both training and inference.
+ Changing the URL normalizer changes model features and requires evaluation.
+ </para>
+ </section>
<section xml:id="tools.langdetect.training">
<title>Training</title>
<para>
diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/BoundedUrlCharSequenceNormalizerTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/BoundedUrlCharSequenceNormalizerTest.java
new file mode 100644
index 000000000..e000b9780
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/BoundedUrlCharSequenceNormalizerTest.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util.normalizer;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.nio.CharBuffer;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class BoundedUrlCharSequenceNormalizerTest {
+
+ private static final BoundedUrlCharSequenceNormalizer NORMALIZER =
+ BoundedUrlCharSequenceNormalizer.getInstance();
+
+ private static String cp(int... codePoints) {
+ return new String(codePoints, 0, codePoints.length);
+ }
+
+ /** Valid http and https URLs from the WHATWG URL test data, removed as a whole. */
+ private static Stream<Arguments> wholeUrls() {
+ return Stream.of(
+ Arguments.of("http://f:21/", " "),
+ Arguments.of("http://f:21/ b", " b"),
+ Arguments.of("http://example.com/%20foo", " "),
+ Arguments.of("http://user:pass@foo.example.com:8080/x?y#z", " "),
+ Arguments.of("http://[::1]/", " "),
+ Arguments.of("http://[1080::8:800:200C:417A]/index.html", " "),
+ Arguments.of("http://example.com/foo\"bar", " \"bar"),
+ Arguments.of("http://example.com/a'b(c){d}!e", " "),
+ Arguments.of("http://example.com/a`b", " "),
+ Arguments.of("http://example.com/foo<bar", " <bar"),
+ Arguments.of("http://a.b.c/", " "),
+ Arguments.of("HTTP://EXAMPLE.COM/", " "),
+ Arguments.of("Https://example.com/", " "),
+ Arguments.of("https://xn--mnchen-3ya.de/pfad", " "),
+ Arguments.of("http://münchen.de/straße", " "),
+ Arguments.of("http://例え.テスト/パス", " "),
+ Arguments.of("http://example.com/" + cp(0x1F600) + "/x", " "),
+ Arguments.of("http://example.com/" + cp(0x1D400), " "));
+ }
+
+ @ParameterizedTest(name = "\"{0}\"")
+ @MethodSource("wholeUrls")
+ void normalizeRemovesWholeHttpUrls(String text, String expected) {
+ Assertions.assertEquals(expected, NORMALIZER.normalize(text));
+ }
+
+ /** URLs with another scheme are kept, and no email address is matched inside them. */
+ @ParameterizedTest
+ @ValueSource(strings = {"git+https://github.com/foo/bar", "blob:https://example.com:443/",
+ "telnet://user:pass@foobar.com:23/", "redis://user:pass@host:6379/0",
+ "rsync://user:pass@host/module", "ftp://ftp.example.com/pub", "ws://example.com/socket",
+ "sc://user:pass@x/", "javascript://example.com/"})
+ void normalizeKeepsOtherSchemes(String text) {
+ Assertions.assertEquals(text, NORMALIZER.normalize(text));
+ }
+
+ /** The scheme needs a left boundary. */
+ private static Stream<Arguments> leftBoundaries() {
+ return Stream.of(
+ Arguments.of("xhttp://example.com/", "xhttp://example.com/"),
+ Arguments.of("1http://example.com/", "1http://example.com/"),
+ Arguments.of("_http://example.com/", "_http://example.com/"),
+ Arguments.of("a.http://example.com/", "a.http://example.com/"),
+ Arguments.of("a-http://example.com/", "a-http://example.com/"),
+ Arguments.of("ähttp://example.com/", "ähttp://example.com/"),
+ Arguments.of("(http://example.com/a)", "( )"),
+ Arguments.of("[http://example.com/a]", "[ ]"),
+ Arguments.of("<http://example.com/a>", "< >"),
+ Arguments.of("\"http://example.com/a\"", "\" \""),
+ Arguments.of("«http://example.com/a»", "« »"),
+ Arguments.of("see:http://example.com/a", "see:http://example.com/a"),
+ Arguments.of("see,http://example.com/a", "see, "),
+ Arguments.of("see/http://example.com/a", "see/ "),
+ Arguments.of("see http://example.com/a", "see "),
+ Arguments.of("\u201chttp://example.com/a\u201d", "\u201c \u201d"),
+ Arguments.of("\u201ehttp://example.com/a\u201c", "\u201e \u201c"));
+ }
+
+ @ParameterizedTest(name = "\"{0}\"")
+ @MethodSource("leftBoundaries")
+ void normalizeNeedsALeftBoundaryBeforeTheScheme(String text, String expected) {
+ Assertions.assertEquals(expected, NORMALIZER.normalize(text));
+ }
+
+ /** Trailing punctuation and unbalanced closing brackets belong to the text. */
+ private static Stream<Arguments> trailingPunctuation() {
+ return Stream.of(
+ Arguments.of("see http://example.com/a.", "see ."),
+ Arguments.of("see http://example.com/a, then", "see , then"),
+ Arguments.of("http://example.com/a?", " ?"),
+ Arguments.of("http://example.com/a!?", " !?"),
+ Arguments.of("http://example.com/a:", " :"),
+ Arguments.of("http://example.com/a;", " ;"),
+ Arguments.of("(http://example.com/a).", "( )."),
+ Arguments.of("http://example.com/a(b)", " "),
+ Arguments.of("http://example.com/a(b).", " ."),
+ Arguments.of("http://example.com/a(b))", " )"),
+ Arguments.of("http://[::1]", " "),
+ Arguments.of("http://example.com/a]", " ]"),
+ Arguments.of("http://example.com/a?b=1#c", " "),
+ Arguments.of("http://example.com/a?b=1&c=2;d", " "),
+ Arguments.of("http://example.com/..", " .."),
+ Arguments.of("http://example.com/a/", " "));
+ }
+
+ @ParameterizedTest(name = "\"{0}\"")
+ @MethodSource("trailingPunctuation")
+ void normalizeGivesBackTrailingPunctuation(String text, String expected) {
+ Assertions.assertEquals(expected, NORMALIZER.normalize(text));
+ }
+
+ /** Whitespace, control characters, delimiters, and unpaired surrogates end the body. */
+ private static Stream<Arguments> bodyEnds() {
+ return Stream.of(
+ Arguments.of("http://example.com/a b", " b"),
+ Arguments.of("http://example.com/a\tb", " \tb"),
+ Arguments.of("http://example.com/a\nb", " \nb"),
+ Arguments.of("http://example.com/a" + cp(0x00A0) + "b", " " + cp(0x00A0) + "b"),
+ Arguments.of("http://example.com/a" + cp(0x3000) + "b", " " + cp(0x3000) + "b"),
+ Arguments.of("http://example.com/a" + cp(0x2028) + "b", " " + cp(0x2028) + "b"),
+ Arguments.of("http://example.com/a" + cp(0x0085) + "b", " " + cp(0x0085) + "b"),
+ Arguments.of("http://example.com/a" + cp(0x0001) + "b", " " + cp(0x0001) + "b"),
+ Arguments.of("http://example.com/a" + cp(0x007F) + "b", " " + cp(0x007F) + "b"),
+ Arguments.of("http://example.com/a" + (char) 0xD800 + "b", " " + (char) 0xD800 + "b"),
+ Arguments.of("http://example.com/a" + (char) 0xDC00, " " + (char) 0xDC00),
+ Arguments.of((char) 0xD800 + "http://example.com/a", (char) 0xD800 + " "),
+ Arguments.of("http://example.com/a" + cp(0x200B) + "b", " "),
+ Arguments.of("http://example.com/a>b", " >b"));
+ }
+
+ @ParameterizedTest(name = "\"{0}\"")
+ @MethodSource("bodyEnds")
+ void normalizeEndsTheBodyAtDelimiters(String text, String expected) {
+ Assertions.assertEquals(expected, NORMALIZER.normalize(text));
+ }
+
+ /** A scheme without a body, or with only punctuation, is no URL. */
+ @ParameterizedTest
+ @ValueSource(strings = {"http://", "https://", "http://?", "http://.", "http:// x", "http:/x",
+ "http:foo", "http//x", "://x", "http", "https:", ""})
+ void normalizeKeepsTextWithoutAUrl(String text) {
+ Assertions.assertEquals(text, NORMALIZER.normalize(text));
+ }
+
+ private static Stream<Arguments> positionsAndCounts() {
+ return Stream.of(
+ Arguments.of("http://a.b/ text", " text"),
+ Arguments.of("text http://a.b/", "text "),
+ Arguments.of("http://a.b/", " "),
+ Arguments.of("http://a.b/ and http://c.d/", " and "),
+ Arguments.of("http://a.b/ http://c.d/", " "),
+ Arguments.of("x http://a.b/ y https://c.d/e z", "x y z"),
+ Arguments.of("http://a.b/?q=http://c.d/", " "));
+ }
+
+ @ParameterizedTest(name = "\"{0}\"")
+ @MethodSource("positionsAndCounts")
+ void normalizeHandlesUrlsAtAnyPosition(String text, String expected) {
+ Assertions.assertEquals(expected, NORMALIZER.normalize(text));
+ }
+
+ private static Stream<Arguments> mailAddresses() {
+ return Stream.of(
+ Arguments.of("asdf asd.fdfa@hasdk23.com.br 2nnfdf", "asdf 2nnfdf"),
+ Arguments.of("asdf asd+fdfa@hasdk23.com.br 2nnfdf", "asdf 2nnfdf"),
+ Arguments.of("asdf asd.fdfa@hasdk23.com_br 2nnfdf", "asdf _br 2nnfdf"),
+ Arguments.of("mail me@example.com or http://example.com/x", "mail or "),
+ Arguments.of("http://user:pass@example.com/x me@example.com", " "),
+ Arguments.of("a@b", "a@b"),
+ Arguments.of("a@bc", " "),
+ Arguments.of("@example.com", "@example.com"));
+ }
+
+ @ParameterizedTest(name = "\"{0}\"")
+ @MethodSource("mailAddresses")
+ void normalizeRemovesMailAddressesOutsideUrls(String text, String expected) {
+ Assertions.assertEquals(expected, NORMALIZER.normalize(text));
+ }
+
+ @Test
+ void normalizeNullThrows() {
+ IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class,
+ () -> NORMALIZER.normalize(null));
+ Assertions.assertEquals("The text must not be null.", e.getMessage());
+ }
+
+ @Test
+ void normalizeReturnsTheInputWhenNothingMatches() {
+ String text = "plain text with no url or address";
+ Assertions.assertSame(text, NORMALIZER.normalize(text));
+ }
+
+ @Test
+ void normalizeAcceptsAnyCharSequence() {
+ String text = "a http://example.com/x b";
+ Assertions.assertEquals("a b", NORMALIZER.normalize(new StringBuilder(text)));
+ Assertions.assertEquals("a b", NORMALIZER.normalize(CharBuffer.wrap(text.toCharArray())));
+ }
+
+ @Test
+ void deserializationYieldsTheSharedInstance() throws Exception {
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ try (ObjectOutputStream out = new ObjectOutputStream(bytes)) {
+ out.writeObject(NORMALIZER);
+ }
+ try (ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) {
+ Assertions.assertSame(NORMALIZER, in.readObject());
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {")", "]", "}", ")]}", ").]!}"})
+ void normalizeLongBracketSuffixWithLinearWork(String suffix) {
+ String tail = suffix.repeat(1000);
+ String input = "http://example.org/" + tail;
+ Assertions.assertEquals(" " + tail, NORMALIZER.normalize(new ReadBudget(input)));
+ }
+
+ @Test
+ void normalizeBalancedBracketsBeforeExcessClosers() {
+ String tail = ")]}".repeat(1000);
+ String input = "http://example.org/(a)[b]{c}" + tail;
+ Assertions.assertEquals(" " + tail, NORMALIZER.normalize(new ReadBudget(input)));
+ }
+
+ /** Counts character accesses so complexity regressions fail without a timing threshold. */
+ private static final class ReadBudget implements CharSequence {
+ private final String text;
+ private int remaining;
+
+ private ReadBudget(String text) {
+ this.text = text;
+ remaining = 32 * text.length();
+ }
+
+ @Override
+ public int length() {
+ return text.length();
+ }
+
+ @Override
+ public char charAt(int index) {
+ Assertions.assertTrue(remaining-- > 0, "Normalization exceeded the linear character-read budget");
+ return text.charAt(index);
+ }
+
+ @Override
+ public CharSequence subSequence(int start, int end) {
+ return text.subSequence(start, end);
+ }
+
+ @Override
+ public String toString() {
+ return text;
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizerTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizerTest.java
index 0093b89a1..ead011d36 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizerTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/UrlCharSequenceNormalizerTest.java
@@ -17,8 +17,13 @@
package opennlp.tools.util.normalizer;
+import java.util.stream.Stream;
+
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
public class UrlCharSequenceNormalizerTest {
@@ -48,4 +53,40 @@ public class UrlCharSequenceNormalizerTest {
Assertions.assertEquals(
"asdf _br 2nnfdf", normalizer.normalize("asdf asd.fdfa@hasdk23.com_br 2nnfdf"));
}
+
+ /**
+ * The output on URLs from the WHATWG URL test data, as language detector models were
+ * trained on it. These rows pin the limits stated in the class Javadoc (OPENNLP-1946);
+ * {@link BoundedUrlCharSequenceNormalizer} is the variant without them.
+ */
+ private static Stream<Arguments> trainedOutput() {
+ return Stream.of(
+ Arguments.of("http://f:21/", " :21/"),
+ Arguments.of("http://example.com/%20foo", " %20foo"),
+ Arguments.of("http://user:pass@foo.example.com:8080/x", " : :8080/x"),
+ Arguments.of("http://[::1]/", "http://[::1]/"),
+ Arguments.of("http://example.com/foo\"bar", " \"bar"),
+ Arguments.of("http://example.com/a'b", " 'b"),
+ Arguments.of("HTTP://EXAMPLE.COM/", "HTTP://EXAMPLE.COM/"),
+ Arguments.of("http://münchen.de/", " ünchen.de/"),
+ Arguments.of("git+https://github.com/foo/bar", "git+ "),
+ Arguments.of("blob:https://example.com:443/", "blob: :443/"),
+ Arguments.of("telnet://user:pass@foobar.com:23/", "telnet://user: :23/"),
+ Arguments.of("redis://user:pass@host:6379/0", "redis://user: :6379/0"),
+ Arguments.of("http://256.256.256.256", " "),
+ Arguments.of("http://?", " "),
+ Arguments.of("see http://example.com/a.", "see "),
+ Arguments.of("(http://example.com/a)", "( )"));
+ }
+
+ @ParameterizedTest(name = "\"{0}\"")
+ @MethodSource("trainedOutput")
+ void normalizeKeepsTheOutputModelsWereTrainedOn(String text, String expected) {
+ Assertions.assertEquals(expected, normalizer.normalize(text));
+ }
+
+ @Test
+ void normalizeNullThrows() {
+ Assertions.assertThrows(IllegalArgumentException.class, () -> normalizer.normalize(null));
+ }
}