You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by rz...@apache.org on 2026/10/09 09:04:51 UTC
(stormcrawler) branch main updated: #2197 BasicURLNormalizer keeps the query when rebuilding a URL (#2207)
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
The following commit(s) were added to refs/heads/main by this push:
new d084f821 #2197 BasicURLNormalizer keeps the query when rebuilding a URL (#2207)
d084f821 is described below
commit d084f82163bcd09c8c736881a9ba43b538e1f944
Author: Julien Nioche <ju...@digitalpebble.com>
AuthorDate: Fri Oct 9 10:04:45 2026 +0100
#2197 BasicURLNormalizer keeps the query when rebuilding a URL (#2207)
* #2197 BasicURLNormalizer keeps the query when rebuilding a URL
When a URL was rebuilt (non-lowercase scheme or host, IDN conversion,
or a path changed by unescapePath/escapePath), the path and query were
passed together as the path to the multi-argument URI constructor. The
'?' was encoded as %3F and existing escapes were double-encoded (%21 ->
%2521).
Only the path is now escaped; the already percent-encoded components
are assembled as is, with the query appended verbatim, and validated
with the single-argument URI constructor.
Fixes #2197
Co-Authored-By: Claude Opus 5.5 <no...@anthropic.com>
* #2197 normalize the query and keep the fragment when rebuilding a URL
- run the query through unescapePath/escapePath separately from the path
- keep the fragment when the URL is rebuilt
- add tests from review: query normalization, host with underscore, anchor
Co-Authored-By: Claude Opus 5.5 <no...@anthropic.com>
* #2197 test: tail comment for the Unicode escapes, as required by Checkstyle
Co-Authored-By: Claude Opus 5.5 <no...@anthropic.com>
---------
Co-authored-by: Claude Opus 5.5 <no...@anthropic.com>
---
.../filtering/basic/BasicURLNormalizer.java | 47 +++++++----
.../filtering/BasicURLNormalizerTest.java | 94 ++++++++++++++++++++++
2 files changed, 125 insertions(+), 16 deletions(-)
diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java
index 341049f3..4e14119e 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java
@@ -31,6 +31,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
@@ -127,7 +128,8 @@ public class BasicURLNormalizer extends URLFilter {
// illegal characters (pipes, backslashes, %uXXXX, etc.) that were
// sanitized during toURL() are reflected in the string we work with
urlToFilter = theUrl.toExternalForm();
- String file = theUrl.getFile();
+ String path = theUrl.getPath();
+ String query = theUrl.getQuery();
String protocol = theUrl.getProtocol();
String host = theUrl.getHost();
boolean hasChanged = !urlToFilter.startsWith(protocol); // lowercased protocol
@@ -151,24 +153,37 @@ public class BasicURLNormalizer extends URLFilter {
}
int port = theUrl.getPort();
- // properly encode characters in path/file using percent-encoding
- String file2 = unescapePath(file);
- file2 = escapePath(file2);
- if (!file.equals(file2)) {
+ // properly encode characters in the path and query using
+ // percent-encoding; they are processed separately so that the
+ // query is not folded into the path when the URL is rebuilt
+ String path2 = unescapePath(path);
+ path2 = escapePath(path2);
+ if (!path.equals(path2)) {
+ hasChanged = true;
+ }
+ String query2 = query == null ? null : escapePath(unescapePath(query));
+ if (!Objects.equals(query, query2)) {
hasChanged = true;
}
if (hasChanged) {
- URI uri =
- new URI(
- protocol,
- null, // userInfo
- host,
- port,
- file2, // path
- null, // query
- null // fragment
- );
- urlToFilter = uri.toString();
+ // the components are already percent-encoded: assemble them
+ // as is rather than using the multi-argument URI constructors,
+ // which would escape '?' and '%' again
+ StringBuilder rebuilt = new StringBuilder(protocol).append(':');
+ if (host != null) {
+ rebuilt.append("//").append(host);
+ if (port != -1) {
+ rebuilt.append(':').append(port);
+ }
+ }
+ rebuilt.append(path2);
+ if (query2 != null) {
+ rebuilt.append('?').append(query2);
+ }
+ if (theUrl.getRef() != null) {
+ rebuilt.append('#').append(theUrl.getRef());
+ }
+ urlToFilter = new URI(rebuilt.toString()).toString();
}
} catch (MalformedURLException | URISyntaxException e) {
return null;
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
index f14d111e..5df8b97a 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
@@ -318,6 +318,100 @@ class BasicURLNormalizerTest {
assertEquals(expectedURL, normalizedUrl, "Failed to filter query string");
}
+ // https://github.com/apache/stormcrawler/issues/2197
+ @Test
+ void testQueryKeptWhenURLRebuilt() throws MalformedURLException {
+ URLFilter urlFilter = createFilter(new ObjectNode(JsonNodeFactory.instance));
+ URL testSourceUrl = URLUtil.toURL("http://a.example/");
+ // rebuilt because of the uppercase scheme and host
+ assertEquals(
+ "http://a.example/p?b=2&a=1",
+ urlFilter.filter(testSourceUrl, new Metadata(), "HTTP://A.Example/p?b=2&a=1"));
+ // rebuilt because the path is unescaped
+ assertEquals(
+ "http://a.example/~user?x=1",
+ urlFilter.filter(testSourceUrl, new Metadata(), "http://a.example/%7euser?x=1"));
+ // rebuilt because an illegal character in the path is escaped
+ assertEquals(
+ "http://a.example/a%5Cb?x=1",
+ urlFilter.filter(testSourceUrl, new Metadata(), "http://a.example/a\\b?x=1"));
+ // port is kept too
+ assertEquals(
+ "http://a.example:8080/p?x=1",
+ urlFilter.filter(testSourceUrl, new Metadata(), "http://A.EXAMPLE:8080/p?x=1"));
+ // not rebuilt: unchanged
+ assertEquals(
+ "http://a.example/p?x=1",
+ urlFilter.filter(testSourceUrl, new Metadata(), "http://a.example/p?x=1"));
+ }
+
+ // https://github.com/apache/stormcrawler/issues/2197
+ @Test
+ void testEscapesNotDoubleEncodedWhenURLRebuilt() throws MalformedURLException {
+ URLFilter urlFilter = createFilter(new ObjectNode(JsonNodeFactory.instance));
+ URL testSourceUrl = URLUtil.toURL("http://a.example/");
+ // existing escapes in the path and query must not become %25XX
+ assertEquals(
+ "http://a.example/a%20b?q=%21x%3Dy",
+ urlFilter.filter(
+ testSourceUrl, new Metadata(), "http://A.example/a%20b?q=%21x%3Dy"));
+
+ urlFilter = createFilter(List.of("utm_source"));
+ assertEquals(
+ "http://a.example/r?t=~%21q&u=https%3A%2F%2Fb.example",
+ urlFilter.filter(
+ testSourceUrl,
+ new Metadata(),
+ "http://A.example/r?u=https%3A%2F%2Fb.example&t=~!q&utm_source=x"));
+ }
+
+ @Test
+ void testQueryNormalizedLikePath() throws MalformedURLException {
+ URLFilter urlFilter = createFilter(new ObjectNode(JsonNodeFactory.instance));
+ URL testSourceUrl = URLUtil.toURL("http://a.example/");
+ // non-ASCII is escaped in the path, so it should be in the query too
+ assertEquals(
+ "http://a.example/%C3%A9?q=%C3%A9",
+ urlFilter.filter(
+ testSourceUrl, new Metadata(), "http://a.example/\u00e9?q=\u00e9")); // é
+ // hex digits of an escape are uppercased
+ assertEquals(
+ "http://a.example/p?q=%E2%84%A2",
+ urlFilter.filter(testSourceUrl, new Metadata(), "http://a.example/p?q=%e2%84%a2"));
+ // escaped unreserved characters are decoded
+ assertEquals(
+ "http://a.example/p?u=~user",
+ urlFilter.filter(testSourceUrl, new Metadata(), "http://a.example/p?u=%7Euser"));
+ }
+
+ @Test
+ void testHostWithUnderscoreWhenURLRebuilt() throws MalformedURLException {
+ URLFilter urlFilter = createFilter(new ObjectNode(JsonNodeFactory.instance));
+ URL testSourceUrl = URLUtil.toURL("http://a.example/");
+ // not rebuilt: passes through
+ assertEquals(
+ "http://a_b.example/p",
+ urlFilter.filter(testSourceUrl, new Metadata(), "http://a_b.example/p"));
+ // rebuilt because of the uppercase host: same result
+ assertEquals(
+ "http://a_b.example/p",
+ urlFilter.filter(testSourceUrl, new Metadata(), "http://A_b.example/p"));
+ }
+
+ @Test
+ void testAnchorKeptWhenURLRebuilt() throws MalformedURLException {
+ URLFilter urlFilter = createFilter(false, false);
+ URL testSourceUrl = URLUtil.toURL("http://a.example/");
+ // not rebuilt: the anchor is kept
+ assertEquals(
+ "http://a.example/p?x=1#sec",
+ urlFilter.filter(testSourceUrl, new Metadata(), "http://a.example/p?x=1#sec"));
+ // rebuilt because of the uppercase host: the anchor should be kept too
+ assertEquals(
+ "http://a.example/p?x=1#sec",
+ urlFilter.filter(testSourceUrl, new Metadata(), "http://A.example/p?x=1#sec"));
+ }
+
private JsonNode getArrayNode(List<String> queryElementsToRemove) {
ObjectMapper mapper = new ObjectMapper();
return mapper.valueToTree(queryElementsToRemove);