You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by rz...@apache.org on 2026/03/24 11:48:29 UTC
(stormcrawler) branch main updated: #1597 - Java 20 deprecates all constructors of the class java.net.URL (#1834)
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
The following commit(s) were added to refs/heads/main by this push:
new bcc73482 #1597 - Java 20 deprecates all constructors of the class java.net.URL (#1834)
bcc73482 is described below
commit bcc7348269da6999866d0e7ce6102b729cf36312
Author: Richard Zowalla <13...@users.noreply.github.com>
AuthorDate: Tue Mar 24 12:48:24 2026 +0100
#1597 - Java 20 deprecates all constructors of the class java.net.URL (#1834)
* #1598 - URL -> URI
1. URLUtil.toURL(String) utility method
- Added URLUtil.toURL(String) that wraps new URI(url).toURL() and converts URISyntaxException into MalformedURLException —
callers no longer need to handle two exception types
- Fixed getHostSegments(String) to use toURL() and removed URISyntaxException from its signature
2. Pre-sanitization in BasicURLNormalizer.sanitizeForURI()
- Encodes common illegal URI characters before parsing: | → %7C, \ → %5C, space → %20, { → %7B, } → %7D
- Converts non-standard %uXXXX percent encoding to proper UTF-8 percent encoding (e.g., %u011f → %C4%9F)
- Applied at the start of filter() so all subsequent URI parsing succeeds
* #1597 - Migrate from deprecated URL constructors to URI-based parsing
Add URLUtil.toURL() and URLUtil.toURI() as drop-in replacements for the
deprecated new URL(String) constructor. Both use a two-phase approach:
strict RFC 3986 parsing first, then fallback sanitization of common
illegal characters (pipes, backslashes, spaces, curly braces, %uXXXX
encoding) to preserve the leniency of the old URL constructor.
Replace all new URI(url).toURL() calls across the codebase with
URLUtil.toURL(). Update BasicURLNormalizer to sync its working string
via toExternalForm() after parsing, eliminating the need for separate
string sanitization.
---
.../apache/stormcrawler/bolt/FeedParserBolt.java | 3 +-
.../org/apache/stormcrawler/bolt/FetcherBolt.java | 5 +-
.../apache/stormcrawler/bolt/JSoupParserBolt.java | 6 +-
.../stormcrawler/bolt/SimpleFetcherBolt.java | 3 +-
.../stormcrawler/bolt/SiteMapParserBolt.java | 3 +-
.../stormcrawler/bolt/URLPartitionerBolt.java | 3 +-
.../apache/stormcrawler/filtering/URLFilters.java | 3 +-
.../filtering/basic/BasicURLNormalizer.java | 30 ++++--
.../stormcrawler/filtering/host/HostURLFilter.java | 3 +-
.../filtering/regex/FastURLFilter.java | 3 +-
.../filtering/robots/RobotsFilter.java | 3 +-
.../stormcrawler/indexing/AbstractIndexerBolt.java | 2 +-
.../apache/stormcrawler/jsoup/LinkParseFilter.java | 5 +-
.../apache/stormcrawler/parse/JSoupFilters.java | 4 +-
.../apache/stormcrawler/parse/ParseFilters.java | 4 +-
.../stormcrawler/parse/filter/LinkParseFilter.java | 5 +-
.../stormcrawler/protocol/RobotRulesParser.java | 3 +-
.../stormcrawler/protocol/file/FileResponse.java | 3 +-
.../stormcrawler/protocol/okhttp/HttpProtocol.java | 5 +-
.../apache/stormcrawler/util/URLPartitioner.java | 4 +-
.../java/org/apache/stormcrawler/util/URLUtil.java | 108 +++++++++++++++++++-
.../stormcrawler/filtering/BasicURLFilterTest.java | 9 +-
.../filtering/BasicURLNormalizerTest.java | 49 ++++-----
.../stormcrawler/filtering/FastURLFilterTest.java | 9 +-
.../stormcrawler/filtering/HostURLFilterTest.java | 9 +-
.../stormcrawler/filtering/MaxDepthFilterTest.java | 9 +-
.../filtering/MetadataFilterFromJsonTest.java | 19 ++--
.../stormcrawler/filtering/MetadataFilterTest.java | 29 +++---
.../stormcrawler/filtering/RegexFilterTest.java | 15 +--
.../stormcrawler/util/CookieConverterTest.java | 2 +-
.../stormcrawler/util/URLUtilSanitizeTest.java | 98 ++++++++++++++++++
.../apache/stormcrawler/util/URLUtilToURLTest.java | 112 +++++++++++++++++++++
.../protocol/selenium/RemoteDriverProtocol.java | 4 +-
.../org/apache/stormcrawler/tika/ParserBolt.java | 2 +-
.../org/apache/stormcrawler/warc/WARCSpout.java | 3 +-
35 files changed, 460 insertions(+), 117 deletions(-)
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java
index 3abe00ff..5d04a80f 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java
@@ -52,6 +52,7 @@ import org.apache.stormcrawler.parse.ParseResult;
import org.apache.stormcrawler.persistence.Status;
import org.apache.stormcrawler.protocol.ProtocolResponse;
import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.URLUtil;
import org.slf4j.LoggerFactory;
import org.xml.sax.InputSource;
@@ -175,7 +176,7 @@ public class FeedParserBolt extends StatusEmitterBolt {
feed = input.build(new InputSource(is));
}
- URL url1 = new URL(url);
+ URL url1 = URLUtil.toURL(url);
List<SyndEntry> entries = feed.getEntries();
for (SyndEntry entry : entries) {
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
index 295631b1..680c6510 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
@@ -61,6 +61,7 @@ import org.apache.stormcrawler.protocol.ProtocolResponse;
import org.apache.stormcrawler.protocol.RobotRules;
import org.apache.stormcrawler.util.ConfUtils;
import org.apache.stormcrawler.util.PerSecondReducer;
+import org.apache.stormcrawler.util.URLUtil;
import org.slf4j.LoggerFactory;
/**
@@ -529,7 +530,7 @@ public class FetcherBolt extends StatusEmitterBolt {
boolean asap = false;
try {
- URL url = new URL(fit.url);
+ URL url = URLUtil.toURL(fit.url);
Protocol protocol = protocolFactory.getProtocol(url);
if (protocol == null) {
@@ -982,7 +983,7 @@ public class FetcherBolt extends StatusEmitterBolt {
URL url;
try {
- url = new URL(urlString);
+ url = URLUtil.toURL(urlString);
} catch (MalformedURLException e) {
LOG.error("{} is a malformed URL", urlString);
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
index 794eb0ed..b412fd9c 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
@@ -292,7 +292,7 @@ public class JSoupParserBolt extends StatusEmitterBolt {
} else {
final Elements links = jsoupDoc.select("a[href]");
slinks = new HashMap<>(links.size());
- final URL baseUrl = new URL(url);
+ final URL baseUrl = URLUtil.toURL(url);
for (Element link : links) {
// nofollow
String[] relkeywords = link.attr("rel").split(" ");
@@ -374,7 +374,7 @@ public class JSoupParserBolt extends StatusEmitterBolt {
// https://github.com/apache/stormcrawler/issues/954
if (allowRedirs() && StringUtils.isNotBlank(redirection)) {
- emitOutlink(tuple, new URL(url), redirection, metadata);
+ emitOutlink(tuple, URLUtil.toURL(url), redirection, metadata);
}
// Mark URL as redirected
@@ -515,7 +515,7 @@ public class JSoupParserBolt extends StatusEmitterBolt {
URL sourceUrl;
try {
- sourceUrl = new URL(url);
+ sourceUrl = URLUtil.toURL(url);
} catch (MalformedURLException e) {
// we would have known by now as previous components check whether
// the URL is valid
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
index e5eb16f5..410f34f9 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
@@ -53,6 +53,7 @@ import org.apache.stormcrawler.protocol.ProtocolResponse;
import org.apache.stormcrawler.protocol.RobotRules;
import org.apache.stormcrawler.util.ConfUtils;
import org.apache.stormcrawler.util.PerSecondReducer;
+import org.apache.stormcrawler.util.URLUtil;
import org.slf4j.LoggerFactory;
/**
@@ -265,7 +266,7 @@ public class SimpleFetcherBolt extends StatusEmitterBolt {
URL url;
try {
- url = new URL(urlString);
+ url = URLUtil.toURL(urlString);
} catch (MalformedURLException e) {
LOG.error("{} is a malformed URL", urlString);
// Report to status stream and ack
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java
index 6736b41f..466fa691 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java
@@ -61,6 +61,7 @@ import org.apache.stormcrawler.parse.ParseResult;
import org.apache.stormcrawler.persistence.DefaultScheduler;
import org.apache.stormcrawler.persistence.Status;
import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.URLUtil;
import org.slf4j.LoggerFactory;
/**
@@ -183,7 +184,7 @@ public class SiteMapParserBolt extends StatusEmitterBolt {
String url, byte[] content, String contentType, Metadata parentMetadata)
throws UnknownFormatException, IOException {
- URL url1 = new URL(url);
+ URL url1 = URLUtil.toURL(url);
long start = System.currentTimeMillis();
AbstractSiteMap siteMap;
// let the parser guess what the mimetype is
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/URLPartitionerBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/URLPartitionerBolt.java
index 8f44779f..ad0deb52 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/URLPartitionerBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/URLPartitionerBolt.java
@@ -36,6 +36,7 @@ import org.apache.storm.tuple.Values;
import org.apache.stormcrawler.Constants;
import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.URLUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -82,7 +83,7 @@ public class URLPartitionerBolt extends BaseRichBolt {
if (partitionKey == null) {
URL u;
try {
- u = new URL(url);
+ u = URLUtil.toURL(url);
host = u.getHost();
} catch (MalformedURLException e1) {
eventCounter.scope("Invalid URL").incrBy(1);
diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/URLFilters.java b/core/src/main/java/org/apache/stormcrawler/filtering/URLFilters.java
index daeec876..1df51a83 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/URLFilters.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/URLFilters.java
@@ -38,6 +38,7 @@ import org.apache.stormcrawler.JSONResource;
import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.util.ConfUtils;
import org.apache.stormcrawler.util.Configurable;
+import org.apache.stormcrawler.util.URLUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.LoggerFactory;
@@ -185,7 +186,7 @@ public class URLFilters extends URLFilter implements JSONResource {
for (URLFilter filter : filters.filters) {
long start = System.currentTimeMillis();
normalizedUrl =
- filter.filter(new URL(sourceUrl), new Metadata(), normalizedUrl);
+ filter.filter(URLUtil.toURL(sourceUrl), new Metadata(), normalizedUrl);
long end = System.currentTimeMillis();
System.out.println(
"\t["
diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java
index 9a5692ef..341049f3 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java
@@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode;
import java.net.IDN;
import java.net.MalformedURLException;
import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
@@ -39,6 +40,7 @@ import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.filtering.URLFilter;
+import org.apache.stormcrawler.util.URLUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
@@ -120,7 +122,11 @@ public class BasicURLNormalizer extends URLFilter {
}
try {
- URL theUrl = new URL(urlToFilter);
+ URL theUrl = URLUtil.toURL(urlToFilter);
+ // sync the string with what the parser produced — this ensures
+ // illegal characters (pipes, backslashes, %uXXXX, etc.) that were
+ // sanitized during toURL() are reflected in the string we work with
+ urlToFilter = theUrl.toExternalForm();
String file = theUrl.getFile();
String protocol = theUrl.getProtocol();
String host = theUrl.getHost();
@@ -152,17 +158,27 @@ public class BasicURLNormalizer extends URLFilter {
hasChanged = true;
}
if (hasChanged) {
- urlToFilter = new URL(protocol, host, port, file2).toString();
+ URI uri =
+ new URI(
+ protocol,
+ null, // userInfo
+ host,
+ port,
+ file2, // path
+ null, // query
+ null // fragment
+ );
+ urlToFilter = uri.toString();
}
- } catch (MalformedURLException e) {
+ } catch (MalformedURLException | URISyntaxException e) {
return null;
}
if (checkValidUri) {
try {
- URI uri = URI.create(urlToFilter);
+ URI uri = URLUtil.toURI(urlToFilter);
urlToFilter = uri.normalize().toString();
- } catch (java.lang.IllegalArgumentException e) {
+ } catch (MalformedURLException e) {
LOG.info("Invalid URI {} from {} ", urlToFilter, originalURL);
return null;
}
@@ -223,7 +239,7 @@ public class BasicURLNormalizer extends URLFilter {
try {
// Handle illegal characters by making a url first
// this will clean illegal characters like |
- final URL url = new URL(urlToFilter);
+ final URL url = URLUtil.toURL(urlToFilter);
String query = url.getQuery();
String path = url.getPath();
@@ -288,7 +304,7 @@ public class BasicURLNormalizer extends URLFilter {
+ ((s = url.getRef()) != null ? '#' + s : "");
} catch (MalformedURLException e) {
- LOG.warn("Invalid urlToFilter {}. {}", urlToFilter, e);
+ LOG.warn("Invalid urlToFilter {}.", urlToFilter, e);
return null;
}
}
diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java
index f7e5055b..645b7b6d 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java
@@ -24,6 +24,7 @@ import java.net.URL;
import java.util.Map;
import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.filtering.URLFilter;
+import org.apache.stormcrawler.util.URLUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -82,7 +83,7 @@ public class HostURLFilter extends URLFilter {
URL url;
try {
- url = new URL(urlToFilter);
+ url = URLUtil.toURL(urlToFilter);
} catch (MalformedURLException e1) {
return null;
}
diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/regex/FastURLFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/regex/FastURLFilter.java
index fdf18af5..26c9afed 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/regex/FastURLFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/regex/FastURLFilter.java
@@ -37,6 +37,7 @@ import org.apache.stormcrawler.JSONResource;
import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.filtering.URLFilter;
import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.URLUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
@@ -211,7 +212,7 @@ class Rules {
* @throws MalformedURLException
*/
public boolean filter(String url, Metadata metadata) throws MalformedURLException {
- URL u = new URL(url);
+ URL u = URLUtil.toURL(url);
// first try the full hostname
String hostname = u.getHost();
diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/robots/RobotsFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/robots/RobotsFilter.java
index 07140c2f..a346ad77 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/robots/RobotsFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/robots/RobotsFilter.java
@@ -27,6 +27,7 @@ import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.filtering.URLFilter;
import org.apache.stormcrawler.protocol.HttpRobotRulesParser;
import org.apache.stormcrawler.protocol.ProtocolFactory;
+import org.apache.stormcrawler.util.URLUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -61,7 +62,7 @@ public class RobotsFilter extends URLFilter {
@NotNull String urlToFilter) {
URL target;
try {
- target = new URL(urlToFilter);
+ target = URLUtil.toURL(urlToFilter);
} catch (MalformedURLException e) {
return null;
}
diff --git a/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java b/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java
index 1881c7ae..f6f7dcea 100644
--- a/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java
@@ -287,7 +287,7 @@ public abstract class AbstractIndexerBolt extends BaseRichBolt {
}
try {
- URL url1 = new URL(url);
+ URL url1 = URLUtil.toURL(url);
URL canonical = URLUtil.resolveUrl(url1, canonicalValue);
String domain = PaidLevelDomain.getPLD(url1.getHost());
diff --git a/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java b/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java
index 1125c374..dff035ab 100644
--- a/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java
@@ -19,7 +19,6 @@ package org.apache.stormcrawler.jsoup;
import com.fasterxml.jackson.databind.JsonNode;
import java.net.MalformedURLException;
-import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -73,11 +72,11 @@ public class LinkParseFilter extends XPathFilter {
java.net.URL sourceUrl;
try {
- sourceUrl = new URL(url);
+ sourceUrl = URLUtil.toURL(url);
} catch (MalformedURLException e1) {
// we would have known by now as previous components check whether
// the URL is valid
- LOG.error("MalformedURLException on {}", url);
+ LOG.error("Malformed URL on {}", url);
return;
}
diff --git a/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java b/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java
index dfaa61e6..b69432d5 100644
--- a/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java
@@ -21,7 +21,6 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.io.InputStream;
-import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
@@ -38,6 +37,7 @@ import org.apache.stormcrawler.JSONResource;
import org.apache.stormcrawler.util.AbstractConfigurable;
import org.apache.stormcrawler.util.ConfUtils;
import org.apache.stormcrawler.util.Configurable;
+import org.apache.stormcrawler.util.URLUtil;
import org.jetbrains.annotations.NotNull;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
@@ -154,7 +154,7 @@ public class JSoupFilters extends AbstractConfigurable implements JSoupFilter, J
String url = cmd.getArgs()[0];
- byte[] content = IOUtils.toByteArray((new URL(url)).openStream());
+ byte[] content = IOUtils.toByteArray(URLUtil.toURL(url).openStream());
Document doc = Jsoup.parse(new String(content, StandardCharsets.UTF_8), url);
diff --git a/core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java b/core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java
index 45f452a9..dd58778d 100644
--- a/core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java
@@ -23,7 +23,6 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.io.InputStream;
-import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
@@ -39,6 +38,7 @@ import org.apache.storm.utils.Utils;
import org.apache.stormcrawler.JSONResource;
import org.apache.stormcrawler.util.ConfUtils;
import org.apache.stormcrawler.util.Configurable;
+import org.apache.stormcrawler.util.URLUtil;
import org.jetbrains.annotations.NotNull;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
@@ -187,7 +187,7 @@ public class ParseFilters extends ParseFilter implements JSONResource {
String url = cmd.getArgs()[0];
- byte[] content = IOUtils.toByteArray((new URL(url)).openStream());
+ byte[] content = IOUtils.toByteArray(URLUtil.toURL(url).openStream());
Document doc = Jsoup.parse(new String(content, StandardCharsets.UTF_8), url);
diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java
index d379e014..0bde0dca 100644
--- a/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java
@@ -19,7 +19,6 @@ package org.apache.stormcrawler.parse.filter;
import com.fasterxml.jackson.databind.JsonNode;
import java.net.MalformedURLException;
-import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -73,11 +72,11 @@ public class LinkParseFilter extends XPathFilter {
java.net.URL sourceUrl;
try {
- sourceUrl = new URL(url);
+ sourceUrl = URLUtil.toURL(url);
} catch (MalformedURLException e1) {
// we would have known by now as previous components check whether
// the URL is valid
- LOG.error("MalformedURLException on {}", url);
+ LOG.error("Malformed URL on {}", url);
return;
}
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java
index 63849eec..20c4b22e 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java
@@ -34,6 +34,7 @@ import java.util.regex.Pattern;
import javax.security.auth.login.Configuration;
import org.apache.storm.Config;
import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.URLUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -200,7 +201,7 @@ public abstract class RobotRulesParser {
public BaseRobotRules getRobotRulesSet(Protocol protocol, String url) {
URL u;
try {
- u = new URL(url);
+ u = URLUtil.toURL(url);
} catch (Exception e) {
return EMPTY_RULES;
}
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java
index e5858f35..80ffc00d 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java
@@ -32,6 +32,7 @@ import org.apache.http.HttpHeaders;
import org.apache.http.HttpStatus;
import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.protocol.ProtocolResponse;
+import org.apache.stormcrawler.util.URLUtil;
import org.slf4j.LoggerFactory;
public class FileResponse {
@@ -51,7 +52,7 @@ public class FileResponse {
content = new byte[0];
statusCode = HttpStatus.SC_INTERNAL_SERVER_ERROR;
- URL url = new URL(u);
+ URL url = URLUtil.toURL(u);
if (!url.getPath().equals(url.getFile())) {
LOG.warn("url.getPath() != url.getFile(): {}.", url);
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
index deae6983..5ddf19c0 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.Proxy;
-import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.security.cert.CertificateException;
import java.util.ArrayList;
@@ -77,6 +76,7 @@ import org.apache.stormcrawler.protocol.ProtocolResponse.TrimmedContentReason;
import org.apache.stormcrawler.proxy.SCProxy;
import org.apache.stormcrawler.util.ConfUtils;
import org.apache.stormcrawler.util.CookieConverter;
+import org.apache.stormcrawler.util.URLUtil;
import org.jetbrains.annotations.NotNull;
import org.slf4j.LoggerFactory;
@@ -271,7 +271,8 @@ public class HttpProtocol extends AbstractHttpProtocol {
return;
}
try {
- final List<Cookie> cookies = CookieConverter.getCookies(cookieStrings, new URL(url));
+ final List<Cookie> cookies =
+ CookieConverter.getCookies(cookieStrings, URLUtil.toURL(url));
for (Cookie c : cookies) {
rb.addHeader("Cookie", c.getName() + "=" + c.getValue());
}
diff --git a/core/src/main/java/org/apache/stormcrawler/util/URLPartitioner.java b/core/src/main/java/org/apache/stormcrawler/util/URLPartitioner.java
index 384c3eeb..6748954a 100644
--- a/core/src/main/java/org/apache/stormcrawler/util/URLPartitioner.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/URLPartitioner.java
@@ -58,9 +58,9 @@ public class URLPartitioner {
if (partitionKey == null) {
URL u;
try {
- u = new URL(url);
+ u = URLUtil.toURL(url);
host = u.getHost();
- } catch (MalformedURLException e1) {
+ } catch (MalformedURLException e) {
LOG.warn("Invalid URL: {}", url);
return null;
}
diff --git a/core/src/main/java/org/apache/stormcrawler/util/URLUtil.java b/core/src/main/java/org/apache/stormcrawler/util/URLUtil.java
index fe3c72f2..e55e7cb1 100644
--- a/core/src/main/java/org/apache/stormcrawler/util/URLUtil.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/URLUtil.java
@@ -20,8 +20,11 @@ package org.apache.stormcrawler.util;
import java.net.IDN;
import java.net.MalformedURLException;
import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.util.Locale;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
/** Utility class for URL analysis. */
@@ -29,6 +32,101 @@ public class URLUtil {
private URLUtil() {}
+ /** Pattern for non-standard {@code %uXXXX} percent encoding (e.g. from JavaScript escape()). */
+ private static final Pattern ILLEGAL_ESCAPE_PATTERN = Pattern.compile("%u([0-9A-Fa-f]{4})");
+
+ /**
+ * Converts a URL string to a {@link URL} object using {@link URI} to avoid the deprecated
+ * {@code new URL(String)} constructor. If strict RFC 3986 parsing fails, common illegal
+ * characters are sanitized and parsing is retried, mimicking the leniency of the old {@code new
+ * URL(String)} constructor.
+ *
+ * @param url the URL string to convert
+ * @return the parsed URL
+ * @throws MalformedURLException if the string is not a valid URL even after sanitization
+ */
+ public static URL toURL(String url) throws MalformedURLException {
+ try {
+ return toURI(url).toURL();
+ } catch (IllegalArgumentException e) {
+ throw (MalformedURLException) new MalformedURLException(e.getMessage()).initCause(e);
+ }
+ }
+
+ /**
+ * Converts a URL string to a {@link URI} object. If strict RFC 3986 parsing fails, common
+ * illegal characters are sanitized and parsing is retried, mimicking the leniency of the old
+ * {@code new URL(String)} constructor.
+ *
+ * @param url the URL string to convert
+ * @return the parsed URI
+ * @throws MalformedURLException if the string is not a valid URI even after sanitization
+ */
+ public static URI toURI(String url) throws MalformedURLException {
+ try {
+ return new URI(url);
+ } catch (URISyntaxException e) {
+ // strict parsing failed — try sanitizing common illegal characters
+ try {
+ return new URI(sanitizeForURI(url));
+ } catch (URISyntaxException e2) {
+ throw (MalformedURLException)
+ new MalformedURLException(e.getMessage()).initCause(e);
+ }
+ }
+ }
+
+ /**
+ * Pre-sanitize a URL string by encoding characters that are illegal in URIs per RFC 3986 but
+ * commonly found in URLs in the wild. Also converts non-standard {@code %uXXXX} percent
+ * encoding to proper UTF-8 percent encoding.
+ */
+ public static String sanitizeForURI(String url) {
+ // Handle non-standard %uXXXX encoding (e.g. from JavaScript escape())
+ final Matcher matcher = ILLEGAL_ESCAPE_PATTERN.matcher(url);
+ if (matcher.find()) {
+ final StringBuilder sb = new StringBuilder();
+ int end = 0;
+ do {
+ sb.append(url, end, matcher.start());
+ final int codePoint = Integer.parseInt(matcher.group(1), 16);
+ for (byte b :
+ new String(Character.toChars(codePoint)).getBytes(StandardCharsets.UTF_8)) {
+ sb.append(String.format(Locale.ROOT, "%%%02X", b & 0xFF));
+ }
+ end = matcher.end();
+ } while (matcher.find());
+ sb.append(url.substring(end));
+ url = sb.toString();
+ }
+
+ // Encode characters that are illegal in URIs but commonly encountered
+ final StringBuilder sb = new StringBuilder(url.length());
+ for (int i = 0; i < url.length(); i++) {
+ final char c = url.charAt(i);
+ switch (c) {
+ case '|':
+ sb.append("%7C");
+ break;
+ case '\\':
+ sb.append("%5C");
+ break;
+ case ' ':
+ sb.append("%20");
+ break;
+ case '{':
+ sb.append("%7B");
+ break;
+ case '}':
+ sb.append("%7D");
+ break;
+ default:
+ sb.append(c);
+ }
+ }
+ return sb.toString();
+ }
+
/**
* Resolve relative URL-s and fix a few java.net.URL errors in handling of URLs with embedded
* params and pure query targets.
@@ -138,7 +236,7 @@ public class URLUtil {
* @throws MalformedURLException
*/
public static String[] getHostSegments(String url) throws MalformedURLException {
- return getHostSegments(new URL(url));
+ return getHostSegments(toURL(url));
}
/**
@@ -149,7 +247,7 @@ public class URLUtil {
*/
public static String getHost(String url) {
try {
- return new URL(url).getHost().toLowerCase(Locale.ROOT);
+ return toURL(url).getHost().toLowerCase(Locale.ROOT);
} catch (MalformedURLException e) {
return null;
}
@@ -167,7 +265,7 @@ public class URLUtil {
// get the full url, and replace the query string with and empty
// string
url = url.toLowerCase(Locale.ROOT);
- String queryStr = new URL(url).getQuery();
+ String queryStr = toURL(url).getQuery();
return (queryStr != null) ? url.replace("?" + queryStr, "") : url;
} catch (MalformedURLException e) {
return null;
@@ -176,7 +274,7 @@ public class URLUtil {
public static String toASCII(String url) {
try {
- URL u = new URL(url);
+ URL u = toURL(url);
URI p =
new URI(
u.getProtocol(),
@@ -195,7 +293,7 @@ public class URLUtil {
public static String toUNICODE(String url) {
try {
- URL u = new URL(url);
+ URL u = toURL(url);
URI p =
new URI(
u.getProtocol(),
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLFilterTest.java
index 6402fd06..ca63bd99 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLFilterTest.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.filtering.basic.BasicURLFilter;
+import org.apache.stormcrawler.util.URLUtil;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -44,10 +45,10 @@ class BasicURLFilterTest {
void testRepetition() throws MalformedURLException {
URLFilter filter = createFilter(-1, 3);
Metadata metadata = new Metadata();
- URL targetURL = new URL("http://www.sourcedomain.com/a/a/a/index.html");
+ URL targetURL = URLUtil.toURL("http://www.sourcedomain.com/a/a/a/index.html");
String filterResult = filter.filter(targetURL, metadata, targetURL.toExternalForm());
Assertions.assertNull(filterResult);
- targetURL = new URL("http://www.sourcedomain.com/a/b/a/index.html");
+ targetURL = URLUtil.toURL("http://www.sourcedomain.com/a/b/a/index.html");
filterResult = filter.filter(targetURL, metadata, targetURL.toExternalForm());
Assertions.assertEquals(targetURL.toExternalForm(), filterResult);
}
@@ -56,10 +57,10 @@ class BasicURLFilterTest {
void testLength() throws MalformedURLException {
URLFilter filter = createFilter(32, -1);
Metadata metadata = new Metadata();
- URL targetURL = new URL("http://www.sourcedomain.com/a/a/a/index.html");
+ URL targetURL = URLUtil.toURL("http://www.sourcedomain.com/a/a/a/index.html");
String filterResult = filter.filter(targetURL, metadata, targetURL.toExternalForm());
Assertions.assertNull(filterResult);
- targetURL = new URL("http://www.sourcedomain.com/");
+ targetURL = URLUtil.toURL("http://www.sourcedomain.com/");
filterResult = filter.filter(targetURL, metadata, targetURL.toExternalForm());
Assertions.assertEquals(targetURL.toExternalForm(), filterResult);
}
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
index 42f260f4..f14d111e 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
@@ -32,6 +32,7 @@ import java.util.Locale;
import java.util.Map;
import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.filtering.basic.BasicURLNormalizer;
+import org.apache.stormcrawler.util.URLUtil;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -82,7 +83,7 @@ class BasicURLNormalizerTest {
@Test
void testAnchorFilter() throws MalformedURLException {
URLFilter allAllowed = createFilter(true, false);
- URL url = new URL("http://www.sourcedomain.com/#0");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/#0");
Metadata metadata = new Metadata();
String filterResult = allAllowed.filter(url, metadata, url.toExternalForm());
String expected = "http://www.sourcedomain.com/";
@@ -92,7 +93,7 @@ class BasicURLNormalizerTest {
@Test
void testAnchorFilterFalse() throws MalformedURLException {
URLFilter allAllowed = createFilter(false, false);
- URL url = new URL("http://www.sourcedomain.com/#0");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/#0");
Metadata metadata = new Metadata();
String filterResult = allAllowed.filter(url, metadata, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
@@ -101,7 +102,7 @@ class BasicURLNormalizerTest {
@Test
void testRemoveSomeOfManyQueryParams() throws MalformedURLException {
URLFilter urlFilter = createFilter(queryParamsToFilter);
- URL testSourceUrl = new URL("http://google.com");
+ URL testSourceUrl = URLUtil.toURL("http://google.com");
String testUrl = "http://google.com?keep1=true&a=c&foo=baz&keep2=true";
String expectedResult = "http://google.com?keep1=true&keep2=true";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -111,7 +112,7 @@ class BasicURLNormalizerTest {
@Test
void testRemoveAllQueryParams() throws MalformedURLException {
URLFilter urlFilter = createFilter(queryParamsToFilter);
- URL testSourceUrl = new URL("http://google.com");
+ URL testSourceUrl = URLUtil.toURL("http://google.com");
String testUrl = "http://google.com?a=c&foo=baz";
String expectedResult = "http://google.com";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -121,7 +122,7 @@ class BasicURLNormalizerTest {
@Test
void testRemoveDupeQueryParams() throws MalformedURLException {
URLFilter urlFilter = createFilter(queryParamsToFilter);
- URL testSourceUrl = new URL("http://google.com");
+ URL testSourceUrl = URLUtil.toURL("http://google.com");
String testUrl = "http://google.com?a=c&foo=baz&foo=bar&test=true";
String expectedResult = "http://google.com?test=true";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -131,7 +132,7 @@ class BasicURLNormalizerTest {
@Test
void testPipeInUrlAndFilterStillWorks() throws MalformedURLException {
URLFilter urlFilter = createFilter(queryParamsToFilter);
- URL testSourceUrl = new URL("http://google.com");
+ URL testSourceUrl = URLUtil.toURL("http://google.com");
String testUrl = "http://google.com?a=c|d&foo=baz&foo=bar&test=true";
String expectedResult = "http://google.com?test=true";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -141,7 +142,7 @@ class BasicURLNormalizerTest {
@Test
void testBothAnchorAndQueryFilter() throws MalformedURLException {
URLFilter urlFilter = createFilter(true, queryParamsToFilter);
- URL testSourceUrl = new URL("http://google.com");
+ URL testSourceUrl = URLUtil.toURL("http://google.com");
String testUrl = "http://google.com?a=c|d&foo=baz&foo=bar&test=true#fragment=ohYeah";
String expectedResult = "http://google.com?test=true";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -151,7 +152,7 @@ class BasicURLNormalizerTest {
@Test
void testQuerySort() throws MalformedURLException {
URLFilter urlFilter = createFilter(queryParamsToFilter);
- URL testSourceUrl = new URL("http://google.com");
+ URL testSourceUrl = URLUtil.toURL("http://google.com");
String testUrl = "http://google.com?a=c|d&foo=baz&foo=bar&test=true&z=2&d=4";
String expectedResult = "http://google.com?d=4&test=true&z=2";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -161,7 +162,7 @@ class BasicURLNormalizerTest {
@Test
void testMangledQueryString() throws MalformedURLException {
URLFilter urlFilter = createFilter(queryParamsToFilter);
- URL testSourceUrl = new URL("http://google.com");
+ URL testSourceUrl = URLUtil.toURL("http://google.com");
String testUrl = "http://google.com&d=4&good=true";
String expectedResult = "http://google.com?d=4&good=true";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -173,7 +174,7 @@ class BasicURLNormalizerTest {
ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance);
filterParams.put("removeHashes", true);
URLFilter urlFilter = createFilter(filterParams);
- URL testSourceUrl = new URL("http://florida-chemical.com");
+ URL testSourceUrl = URLUtil.toURL("http://florida-chemical.com");
String in =
"http://www.florida-chemical.com/Diacetone-Alcohol-DAA-99.html?xid_0b629=12854b827878df26423d933a5baf86d5";
String out = "http://www.florida-chemical.com/Diacetone-Alcohol-DAA-99.html";
@@ -189,7 +190,7 @@ class BasicURLNormalizerTest {
@Test
void testDontFixMangledQueryString() throws MalformedURLException {
URLFilter urlFilter = createFilter(true, false, queryParamsToFilter);
- URL testSourceUrl = new URL("http://google.com");
+ URL testSourceUrl = URLUtil.toURL("http://google.com");
String testUrl = "http://google.com&d=4&good=true";
String expectedResult = "http://google.com&d=4&good=true";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -199,12 +200,12 @@ class BasicURLNormalizerTest {
@Test
void testFixMangledQueryString() throws MalformedURLException {
URLFilter urlFilter = createFilter(false, true, queryParamsToFilter);
- URL testSourceUrl = new URL("http://google.com");
+ URL testSourceUrl = URLUtil.toURL("http://google.com");
String testUrl = "http://google.com&d=4&good=true";
String expectedResult = "http://google.com?d=4&good=true";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
assertEquals(expectedResult, normalizedUrl, "Failed to filter query string");
- testSourceUrl = new URL("http://dev.com");
+ testSourceUrl = URLUtil.toURL("http://dev.com");
testUrl = "http://dev.com/s&utax/NEWSRLSEfy18.pdf";
normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
expectedResult = "http://dev.com/s&utax/NEWSRLSEfy18.pdf";
@@ -216,7 +217,7 @@ class BasicURLNormalizerTest {
URLFilter urlFilter = createFilter(queryParamsToFilter);
String urlWithEscapedCharacters =
"http://www.dillards.com/product/ASICS-Womens-GT2000-3-LiteShow%E2%84%A2-Running-Shoes_301_-1_301_504736989";
- URL testSourceUrl = new URL(urlWithEscapedCharacters);
+ URL testSourceUrl = URLUtil.toURL(urlWithEscapedCharacters);
String testUrl = urlWithEscapedCharacters;
String expectedResult = urlWithEscapedCharacters;
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -228,7 +229,7 @@ class BasicURLNormalizerTest {
URLFilter urlFilter = createFilter(queryParamsToFilter);
String urlWithEscapedCharacters =
"http://www.dillards.com/product/ASICS-Womens-GT2000-3-LiteShow%E2%84%A2-Running-Shoes_301_-1_301_504736989?how=are&you=doing";
- URL testSourceUrl = new URL(urlWithEscapedCharacters);
+ URL testSourceUrl = URLUtil.toURL(urlWithEscapedCharacters);
String testUrl = urlWithEscapedCharacters;
String expectedResult = urlWithEscapedCharacters;
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -242,7 +243,7 @@ class BasicURLNormalizerTest {
"http://www.voltaix.com/\\SDS\\Silicon\\Trisilane\\Trisilane_SI050_USENG.pdf";
String expectedResult =
"http://www.voltaix.com/%5CSDS%5CSilicon%5CTrisilane%5CTrisilane_SI050_USENG.pdf";
- URL testSourceUrl = new URL(urlWithEscapedCharacters);
+ URL testSourceUrl = URLUtil.toURL("http://www.voltaix.com/");
String testUrl = urlWithEscapedCharacters;
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
assertEquals(expectedResult, normalizedUrl, "Failed to filter query string");
@@ -251,17 +252,17 @@ class BasicURLNormalizerTest {
@Test
void testInvalidURI() throws MalformedURLException {
URLFilter urlFilter = createFilter(true, true);
- // this one is now handled by the normaliser
+ URL testSourceUrl = URLUtil.toURL("http://www.quanjing.com/");
+ // this one is now handled by the normaliser via sanitizeForURI
String nonURI = "http://www.quanjing.com/search.aspx?q=top-651451||1|60|1|2||||&Fr=4";
- URL testSourceUrl = new URL(nonURI);
String expectedResult =
"http://www.quanjing.com/search.aspx?q=top-651451%7C%7C1%7C60%7C1%7C2%7C%7C%7C%7C&Fr=4";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), nonURI);
assertEquals(expectedResult, normalizedUrl, "Failed to filter query string");
- // this one is
+ // this one has spaces in query values
+ testSourceUrl = URLUtil.toURL("http://vins.lemonde.fr/");
nonURI =
"http://vins.lemonde.fr?utm_source=LeMonde_partenaire_hp&utm_medium=EMPLACEMENT PARTENAIRE&utm_term=&utm_content=&utm_campaign=LeMonde_partenaire_hp";
- testSourceUrl = new URL(nonURI);
expectedResult =
"http://vins.lemonde.fr?utm_source=LeMonde_partenaire_hp&utm_medium=EMPLACEMENT%20PARTENAIRE&utm_term=&utm_content=&utm_campaign=LeMonde_partenaire_hp";
normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), nonURI);
@@ -270,7 +271,7 @@ class BasicURLNormalizerTest {
// http://docs.oracle.com/javase/7/docs/api/java/net/URI.html#normalize()
String nonNormURL =
"http://docs.oracle.com/javase/7/docs/api/java/net/../net/./URI.html#normalize()";
- testSourceUrl = new URL(nonNormURL);
+ testSourceUrl = URLUtil.toURL("http://docs.oracle.com/");
expectedResult = "http://docs.oracle.com/javase/7/docs/api/java/net/URI.html";
normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), nonNormURL);
assertEquals(expectedResult, normalizedUrl, "Failed to filter query string");
@@ -279,7 +280,7 @@ class BasicURLNormalizerTest {
@Test
void testLowerCasing() throws MalformedURLException {
URLFilter urlFilter = createFilter(false, false);
- URL testSourceUrl = new URL("http://blablabla.org/");
+ URL testSourceUrl = URLUtil.toURL("http://blablabla.org/");
String inputURL = "HTTP://www.quanjing.com/";
String expectedResult = inputURL.toLowerCase(Locale.ROOT);
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), inputURL);
@@ -294,7 +295,7 @@ class BasicURLNormalizerTest {
@Test
void testNonStandardPercentEncoding() throws MalformedURLException {
URLFilter urlFilter = createFilter(false, false);
- URL testSourceUrl = new URL("http://www.hurriyet.com.tr/index/?d=20160328&p=13");
+ URL testSourceUrl = URLUtil.toURL("http://www.hurriyet.com.tr/index/?d=20160328&p=13");
String inputURL = "http://www.hurriyet.com.tr/index/?d=20160328&p=13&s=ni%u011fde";
String expectedURL = "http://www.hurriyet.com.tr/index/?d=20160328&p=13&s=ni%C4%9Fde";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), inputURL);
@@ -306,7 +307,7 @@ class BasicURLNormalizerTest {
ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance);
filterParams.put("hostIDNtoASCII", true);
URLFilter urlFilter = createFilter(filterParams);
- URL testSourceUrl = new URL("http://www.example.com/");
+ URL testSourceUrl = URLUtil.toURL("http://www.example.com/");
String inputURL = "http://señal6.com.ar/";
String expectedURL = "http://xn--seal6-pta.com.ar/";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), inputURL);
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/FastURLFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/FastURLFilterTest.java
index b1bccd0e..1d3ecffa 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/FastURLFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/FastURLFilterTest.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.filtering.regex.FastURLFilter;
+import org.apache.stormcrawler.util.URLUtil;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -41,7 +42,7 @@ class FastURLFilterTest {
@Test
void testImagesFilter() throws MalformedURLException {
- URL url = new URL("http://www.somedomain.com/image.jpg");
+ URL url = URLUtil.toURL("http://www.somedomain.com/image.jpg");
Metadata metadata = new Metadata();
String filterResult = createFilter().filter(url, metadata, url.toExternalForm());
Assertions.assertNull(filterResult);
@@ -49,19 +50,19 @@ class FastURLFilterTest {
@Test
void testDomainNotAllowed() throws MalformedURLException {
- URL url = new URL("http://stormcrawler.net/");
+ URL url = URLUtil.toURL("http://stormcrawler.net/");
Metadata metadata = new Metadata();
String filterResult = createFilter().filter(url, metadata, url.toExternalForm());
Assertions.assertNull(filterResult);
// allowed
- url = new URL("http://stormcrawler.net/bla/");
+ url = URLUtil.toURL("http://stormcrawler.net/bla/");
filterResult = createFilter().filter(url, metadata, url.toExternalForm());
Assertions.assertEquals(url.toString(), filterResult);
}
@Test
void testMD() throws MalformedURLException {
- URL url = new URL("http://somedomain.net/");
+ URL url = URLUtil.toURL("http://somedomain.net/");
Metadata metadata = new Metadata();
metadata.addValue("key", "value");
String filterResult = createFilter().filter(url, metadata, url.toExternalForm());
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
index b0f00d84..f70ca90b 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.filtering.host.HostURLFilter;
+import org.apache.stormcrawler.util.URLUtil;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -47,7 +48,7 @@ class HostURLFilterTest {
@Test
void testAllAllowed() throws MalformedURLException {
HostURLFilter allAllowed = createFilter(false, false);
- URL sourceURL = new URL("http://www.sourcedomain.com/index.html");
+ URL sourceURL = URLUtil.toURL("http://www.sourcedomain.com/index.html");
Metadata metadata = new Metadata();
String filterResult =
allAllowed.filter(sourceURL, metadata, "http://www.sourcedomain.com/index.html");
@@ -63,7 +64,7 @@ class HostURLFilterTest {
@Test
void testAllForbidden() throws MalformedURLException {
HostURLFilter allAllowed = createFilter(true, true);
- URL sourceURL = new URL("http://www.sourcedomain.com/index.html");
+ URL sourceURL = URLUtil.toURL("http://www.sourcedomain.com/index.html");
Metadata metadata = new Metadata();
String filterResult =
allAllowed.filter(sourceURL, metadata, "http://www.sourcedomain.com/index.html");
@@ -79,7 +80,7 @@ class HostURLFilterTest {
@Test
void testWithinHostOnly() throws MalformedURLException {
HostURLFilter allAllowed = createFilter(true, false);
- URL sourceURL = new URL("http://www.sourcedomain.com/index.html");
+ URL sourceURL = URLUtil.toURL("http://www.sourcedomain.com/index.html");
Metadata metadata = new Metadata();
String filterResult =
allAllowed.filter(sourceURL, metadata, "http://www.sourcedomain.com/index.html");
@@ -95,7 +96,7 @@ class HostURLFilterTest {
@Test
void testWithinDomain() throws MalformedURLException {
HostURLFilter allAllowed = createFilter(false, true);
- URL sourceURL = new URL("http://www.sourcedomain.com/index.html");
+ URL sourceURL = URLUtil.toURL("http://www.sourcedomain.com/index.html");
Metadata metadata = new Metadata();
String filterResult =
allAllowed.filter(sourceURL, metadata, "http://www.sourcedomain.com/index.html");
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/MaxDepthFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/MaxDepthFilterTest.java
index 46489b66..3157fa86 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/MaxDepthFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/MaxDepthFilterTest.java
@@ -26,6 +26,7 @@ import java.util.Map;
import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.filtering.depth.MaxDepthFilter;
import org.apache.stormcrawler.util.MetadataTransfer;
+import org.apache.stormcrawler.util.URLUtil;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -43,7 +44,7 @@ class MaxDepthFilterTest {
@Test
void testDepthZero() throws MalformedURLException {
URLFilter filter = createFilter("maxDepth", 0);
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
String filterResult = filter.filter(url, metadata, url.toExternalForm());
Assertions.assertNull(filterResult);
@@ -52,7 +53,7 @@ class MaxDepthFilterTest {
@Test
void testDepth() throws MalformedURLException {
URLFilter filter = createFilter("maxDepth", 2);
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
metadata.setValue(MetadataTransfer.depthKeyName, "2");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -62,7 +63,7 @@ class MaxDepthFilterTest {
@Test
void testCustomDepthZero() throws MalformedURLException {
URLFilter filter = createFilter("maxDepth", 3);
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
metadata.setValue(MetadataTransfer.maxDepthKeyName, "0");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -72,7 +73,7 @@ class MaxDepthFilterTest {
@Test
void testCustomDepth() throws MalformedURLException {
URLFilter filter = createFilter("maxDepth", 1);
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
metadata.setValue(MetadataTransfer.maxDepthKeyName, "2");
metadata.setValue(MetadataTransfer.depthKeyName, "1");
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterFromJsonTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterFromJsonTest.java
index 62e925de..01e35062 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterFromJsonTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterFromJsonTest.java
@@ -21,6 +21,7 @@ import java.net.MalformedURLException;
import java.net.URL;
import java.util.Map;
import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.util.URLUtil;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -34,7 +35,7 @@ class MetadataFilterFromJsonTest {
@Test
void testFilterNoMD() throws MalformedURLException {
URLFilters filter = createURLFilters("test.metadata.1.urlfilters.json");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
String filterResult = filter.filter(url, metadata, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
@@ -43,7 +44,7 @@ class MetadataFilterFromJsonTest {
@Test
void testFilterHit() throws MalformedURLException {
URLFilters filter = createURLFilters("test.metadata.1.urlfilters.json");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -53,7 +54,7 @@ class MetadataFilterFromJsonTest {
@Test
void testFilterNoHit() throws MalformedURLException {
URLFilters filter = createURLFilters("test.metadata.1.urlfilters.json");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
metadata.addValue("key", "val2");
metadata.addValue("key", "val3");
@@ -65,7 +66,7 @@ class MetadataFilterFromJsonTest {
@Test
void testNewFilterWithEmptyFilterAndNullMetadata() throws MalformedURLException {
URLFilters filter = createURLFilters("test.metadata.2.urlfilters.json");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
String filterResult = filter.filter(url, null, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
}
@@ -73,7 +74,7 @@ class MetadataFilterFromJsonTest {
@Test
void testNewFilterWithEmptyFilterAndEmptyMetadata() throws MalformedURLException {
URLFilters filter = createURLFilters("test.metadata.2.urlfilters.json");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
String filterResult = filter.filter(url, metadata, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
@@ -82,7 +83,7 @@ class MetadataFilterFromJsonTest {
@Test
void testNewFilterWithEmptyMetadata() throws MalformedURLException {
URLFilters filter = createURLFilters("test.metadata.2.urlfilters.json");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
String filterResult = filter.filter(url, metadata, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
@@ -92,7 +93,7 @@ class MetadataFilterFromJsonTest {
void testNewFilterWithOnlyOneMatchingANDFilter() throws MalformedURLException {
// Filter if key=>val AND key2=>val2 match
URLFilters filter = createURLFilters("test.metadata.2.urlfilters.json");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -103,7 +104,7 @@ class MetadataFilterFromJsonTest {
void testNewFilterWithAllMatchingANDFilter() throws MalformedURLException {
// Filter if key=>val AND key2=>val2 match
URLFilters filter = createURLFilters("test.metadata.2.urlfilters.json");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
metadata.addValue("key2", "val2");
@@ -115,7 +116,7 @@ class MetadataFilterFromJsonTest {
void testNewFilterWithComplexFilter() throws MalformedURLException {
// Filter if key=>val AND (key2=>val2 OR key3=>val3) match
URLFilters filter = createURLFilters("test.metadata.3.urlfilters.json");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterTest.java
index 0d7efb41..6d52adb0 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterTest.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.filtering.metadata.MetadataFilter;
+import org.apache.stormcrawler.util.URLUtil;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -43,7 +44,7 @@ class MetadataFilterTest {
@Test
void testFilterNoMD() throws MalformedURLException {
URLFilter filter = createFilter("key", "val");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
String filterResult = filter.filter(url, metadata, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
@@ -52,7 +53,7 @@ class MetadataFilterTest {
@Test
void testFilterHit() throws MalformedURLException {
URLFilter filter = createFilter("key", "val");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -62,7 +63,7 @@ class MetadataFilterTest {
@Test
void testFilterNoHit() throws MalformedURLException {
URLFilter filter = createFilter("key", "val");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
metadata.addValue("key", "val2");
metadata.addValue("key", "val3");
@@ -74,7 +75,7 @@ class MetadataFilterTest {
@Test
void testNewFilterWithEmptyFilterAndNullMetadata() throws MalformedURLException {
MetadataFilter filter = new MetadataFilter();
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
String filterResult = filter.filter(url, null, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
}
@@ -82,7 +83,7 @@ class MetadataFilterTest {
@Test
void testNewFilterWithEmptyFilterAndEmptyMetadata() throws MalformedURLException {
MetadataFilter filter = new MetadataFilter();
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
String filterResult = filter.filter(url, metadata, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
@@ -91,7 +92,7 @@ class MetadataFilterTest {
@Test
void testNewFilterWithEmptyFilter() throws MalformedURLException {
MetadataFilter filter = new MetadataFilter();
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -102,7 +103,7 @@ class MetadataFilterTest {
void testNewFilterWithEmptyMetadata() throws MalformedURLException {
MetadataFilter filter = new MetadataFilter();
filter.addFilter("key", "val");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
String filterResult = filter.filter(url, metadata, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
@@ -113,7 +114,7 @@ class MetadataFilterTest {
// Filter if key=>val match (OR operation)
MetadataFilter filter = new MetadataFilter();
filter.addFilter("key", "val");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -126,7 +127,7 @@ class MetadataFilterTest {
MetadataFilter filter = new MetadataFilter();
filter.addFilter("key", "val");
filter.setOperation(MetadataFilter.FilterOperation.AND);
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -139,7 +140,7 @@ class MetadataFilterTest {
MetadataFilter filter = new MetadataFilter();
filter.addFilter("key", "val");
filter.addFilter("key2", "val2");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -153,7 +154,7 @@ class MetadataFilterTest {
filter.addFilter("key", "val");
filter.addFilter("key2", "val2");
filter.setOperation(MetadataFilter.FilterOperation.AND);
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -167,7 +168,7 @@ class MetadataFilterTest {
filter.addFilter("key", "val");
filter.addFilter("key2", "val2");
filter.setOperation(MetadataFilter.FilterOperation.AND);
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
metadata.addValue("key2", "val2");
@@ -185,7 +186,7 @@ class MetadataFilterTest {
filter2.addFilter("key2", "val2");
filter2.addFilter("key3", "val3");
filter.addFilter(filter2);
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
@@ -231,7 +232,7 @@ class MetadataFilterTest {
filter2.addFilter("key3", "val3");
filter2.setOperation(MetadataFilter.FilterOperation.AND);
filter.addFilter(filter2);
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = URLUtil.toURL("http://www.sourcedomain.com/");
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/RegexFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/RegexFilterTest.java
index 06bbfbe6..42a9e58a 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/RegexFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/RegexFilterTest.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.filtering.regex.RegexURLFilter;
+import org.apache.stormcrawler.util.URLUtil;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -46,7 +47,7 @@ class RegexFilterTest {
@Test
void testProtocolFilter() throws MalformedURLException {
URLFilter allAllowed = createFilter();
- URL url = new URL("ftp://www.someFTP.com/#0");
+ URL url = URLUtil.toURL("ftp://www.someFTP.com/#0");
Metadata metadata = new Metadata();
String filterResult = allAllowed.filter(url, metadata, url.toExternalForm());
Assertions.assertNull(filterResult);
@@ -55,24 +56,24 @@ class RegexFilterTest {
@Test
void testImagesFilter() throws MalformedURLException {
URLFilter allAllowed = createFilter();
- URL url = new URL("http://www.someFTP.com/bla.gif");
+ URL url = URLUtil.toURL("http://www.someFTP.com/bla.gif");
Metadata metadata = new Metadata();
String filterResult = allAllowed.filter(url, metadata, url.toExternalForm());
Assertions.assertNull(filterResult);
- url = new URL("http://www.someFTP.com/bla.GIF");
+ url = URLUtil.toURL("http://www.someFTP.com/bla.GIF");
filterResult = allAllowed.filter(url, metadata, url.toExternalForm());
Assertions.assertNull(filterResult);
- url = new URL("http://www.someFTP.com/bla.GIF&somearg=0");
+ url = URLUtil.toURL("http://www.someFTP.com/bla.GIF&somearg=0");
filterResult = allAllowed.filter(url, metadata, url.toExternalForm());
Assertions.assertNull(filterResult);
- url = new URL("http://www.someFTP.com/bla.GIF?somearg=0");
+ url = URLUtil.toURL("http://www.someFTP.com/bla.GIF?somearg=0");
filterResult = allAllowed.filter(url, metadata, url.toExternalForm());
Assertions.assertNull(filterResult);
// not this one : the gif is within the path
- url = new URL("http://www.someFTP.com/bla.GIF.orNot");
+ url = URLUtil.toURL("http://www.someFTP.com/bla.GIF.orNot");
filterResult = allAllowed.filter(url, metadata, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
- url = new URL("http://www.someFTP.com/bla.mp4");
+ url = URLUtil.toURL("http://www.someFTP.com/bla.mp4");
filterResult = allAllowed.filter(url, metadata, url.toExternalForm());
Assertions.assertNull(filterResult);
}
diff --git a/core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java b/core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java
index 0c6dfbca..57b95f45 100644
--- a/core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java
@@ -308,7 +308,7 @@ class CookieConverterTest {
private URL getUrl(String urlString) {
try {
- return new URL(urlString);
+ return URLUtil.toURL(urlString);
} catch (MalformedURLException e) {
return null;
}
diff --git a/core/src/test/java/org/apache/stormcrawler/util/URLUtilSanitizeTest.java b/core/src/test/java/org/apache/stormcrawler/util/URLUtilSanitizeTest.java
new file mode 100644
index 00000000..95fc7737
--- /dev/null
+++ b/core/src/test/java/org/apache/stormcrawler/util/URLUtilSanitizeTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.util;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+
+class URLUtilSanitizeTest {
+
+ @Test
+ void testSanitizePipe() {
+ String input = "http://example.com?q=a|b";
+ String result = URLUtil.sanitizeForURI(input);
+ assertEquals("http://example.com?q=a%7Cb", result);
+ }
+
+ @Test
+ void testSanitizeBackslash() {
+ String input = "http://example.com/\\path\\file.pdf";
+ String result = URLUtil.sanitizeForURI(input);
+ assertEquals("http://example.com/%5Cpath%5Cfile.pdf", result);
+ }
+
+ @Test
+ void testSanitizeSpace() {
+ String input = "http://example.com/my path/file name.html";
+ String result = URLUtil.sanitizeForURI(input);
+ assertEquals("http://example.com/my%20path/file%20name.html", result);
+ }
+
+ @Test
+ void testSanitizeCurlyBraces() {
+ String input = "http://example.com/{id}/page";
+ String result = URLUtil.sanitizeForURI(input);
+ assertEquals("http://example.com/%7Bid%7D/page", result);
+ }
+
+ @Test
+ void testSanitizeNonStandardPercentEncoding() {
+ // %u011f is non-standard encoding for ğ (U+011F)
+ // Should be converted to UTF-8 percent encoding: %C4%9F
+ String input = "http://example.com/page?s=ni%u011fde";
+ String result = URLUtil.sanitizeForURI(input);
+ assertEquals("http://example.com/page?s=ni%C4%9Fde", result);
+ }
+
+ @Test
+ void testSanitizeNonStandardPercentEncodingMultiple() {
+ String input = "http://example.com/%u0041%u0042";
+ String result = URLUtil.sanitizeForURI(input);
+ // U+0041 = 'A' → %41, U+0042 = 'B' → %42
+ assertEquals("http://example.com/%41%42", result);
+ }
+
+ @Test
+ void testSanitizeValidUrlUnchanged() {
+ String input = "http://example.com/path?query=value&other=123";
+ String result = URLUtil.sanitizeForURI(input);
+ assertEquals(input, result);
+ }
+
+ @Test
+ void testSanitizeAlreadyEncodedUnchanged() {
+ String input = "http://example.com/path%20with%7Cpipe";
+ String result = URLUtil.sanitizeForURI(input);
+ assertEquals(input, result);
+ }
+
+ @Test
+ void testSanitizeMultiplePipes() {
+ String input = "http://example.com?q=top-651451||1|60|1|2||||";
+ String result = URLUtil.sanitizeForURI(input);
+ assertEquals("http://example.com?q=top-651451%7C%7C1%7C60%7C1%7C2%7C%7C%7C%7C", result);
+ }
+
+ @Test
+ void testSanitizeMixedIllegalChars() {
+ String input = "http://example.com/path with|pipe";
+ String result = URLUtil.sanitizeForURI(input);
+ assertEquals("http://example.com/path%20with%7Cpipe", result);
+ }
+}
diff --git a/core/src/test/java/org/apache/stormcrawler/util/URLUtilToURLTest.java b/core/src/test/java/org/apache/stormcrawler/util/URLUtilToURLTest.java
new file mode 100644
index 00000000..81b06bfc
--- /dev/null
+++ b/core/src/test/java/org/apache/stormcrawler/util/URLUtilToURLTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.util;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import org.junit.jupiter.api.Test;
+
+class URLUtilToURLTest {
+
+ @Test
+ void testValidHttpUrl() throws MalformedURLException {
+ URL url = URLUtil.toURL("http://example.com/path?query=value");
+ assertNotNull(url);
+ assertEquals("http", url.getProtocol());
+ assertEquals("example.com", url.getHost());
+ assertEquals("/path", url.getPath());
+ assertEquals("query=value", url.getQuery());
+ }
+
+ @Test
+ void testValidHttpsUrl() throws MalformedURLException {
+ URL url = URLUtil.toURL("https://example.com/");
+ assertNotNull(url);
+ assertEquals("https", url.getProtocol());
+ }
+
+ @Test
+ void testUrlWithPort() throws MalformedURLException {
+ URL url = URLUtil.toURL("http://example.com:8080/path");
+ assertEquals(8080, url.getPort());
+ }
+
+ @Test
+ void testUrlWithFragment() throws MalformedURLException {
+ URL url = URLUtil.toURL("http://example.com/page#section");
+ assertNotNull(url);
+ assertEquals("section", url.getRef());
+ }
+
+ @Test
+ void testPipeFallbackSanitization() throws MalformedURLException {
+ // Pipe is illegal in URIs per RFC 3986 but toURL sanitizes on fallback
+ URL url = URLUtil.toURL("http://example.com?q=a|b");
+ assertNotNull(url);
+ assertEquals("example.com", url.getHost());
+ }
+
+ @Test
+ void testBackslashFallbackSanitization() throws MalformedURLException {
+ // Backslash is illegal in URIs but toURL sanitizes on fallback
+ URL url = URLUtil.toURL("http://example.com/\\path");
+ assertNotNull(url);
+ }
+
+ @Test
+ void testSpaceFallbackSanitization() throws MalformedURLException {
+ // Unencoded space is illegal in URIs but toURL sanitizes on fallback
+ URL url = URLUtil.toURL("http://example.com/my path");
+ assertNotNull(url);
+ }
+
+ @Test
+ void testCompletelyInvalidString() {
+ assertThrows(MalformedURLException.class, () -> URLUtil.toURL("not a url at all"));
+ }
+
+ @Test
+ void testNonAbsoluteUri() {
+ assertThrows(MalformedURLException.class, () -> URLUtil.toURL("just/a/path"));
+ }
+
+ @Test
+ void testEncodedCharactersPreserved() throws MalformedURLException {
+ URL url = URLUtil.toURL("http://example.com/path%20with%20spaces");
+ assertNotNull(url);
+ assertEquals("/path%20with%20spaces", url.getPath());
+ }
+
+ @Test
+ void testUrlWithUserInfo() throws MalformedURLException {
+ URL url = URLUtil.toURL("http://user:pass@example.com/path");
+ assertNotNull(url);
+ assertEquals("user:pass", url.getUserInfo());
+ }
+
+ @Test
+ void testCauseIsPreserved() {
+ MalformedURLException ex =
+ assertThrows(MalformedURLException.class, () -> URLUtil.toURL("not a url at all"));
+ assertNotNull(ex.getCause());
+ }
+}
diff --git a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java b/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java
index 109dbbb1..9f7faa3e 100644
--- a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java
+++ b/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java
@@ -17,7 +17,6 @@
package org.apache.stormcrawler.protocol.selenium;
-import java.net.URL;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
@@ -26,6 +25,7 @@ import java.util.Map.Entry;
import org.apache.storm.Config;
import org.apache.stormcrawler.protocol.Protocol;
import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.URLUtil;
import org.openqa.selenium.WebDriver.Timeouts;
import org.openqa.selenium.remote.DesiredCapabilities;
import org.openqa.selenium.remote.RemoteWebDriver;
@@ -101,7 +101,7 @@ public class RemoteDriverProtocol extends SeleniumProtocol {
for (String cdaddress : addresses) {
try {
RemoteWebDriver driver =
- new RemoteWebDriver(new URL(cdaddress), capabilities, tracing);
+ new RemoteWebDriver(URLUtil.toURL(cdaddress), capabilities, tracing);
// setting timouts
// see https://www.browserstack.com/guide/understanding-selenium-timeouts
Timeouts touts = driver.manage().timeouts();
diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
index d7a40526..2586114a 100644
--- a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
+++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
@@ -205,7 +205,7 @@ public class ParserBolt extends BaseRichBolt {
// as well as the filename
try {
- URL _url = new URL(url);
+ URL _url = URLUtil.toURL(url);
md.set(TikaCoreProperties.RESOURCE_NAME_KEY, _url.getFile());
} catch (MalformedURLException e1) {
throw new IllegalStateException("Malformed URL", e1);
diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCSpout.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCSpout.java
index afde9c85..87308846 100644
--- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCSpout.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCSpout.java
@@ -43,6 +43,7 @@ import org.apache.stormcrawler.persistence.Status;
import org.apache.stormcrawler.protocol.ProtocolResponse;
import org.apache.stormcrawler.spout.FileSpout;
import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.URLUtil;
import org.netpreserve.jwarc.HttpMessage;
import org.netpreserve.jwarc.HttpRequest;
import org.netpreserve.jwarc.HttpResponse;
@@ -150,7 +151,7 @@ public class WARCSpout extends FileSpout {
private ReadableByteChannel openChannel(String path) throws IOException {
if (path.matches("^https?://.*")) {
- URL warcUrl = new URL(path);
+ URL warcUrl = URLUtil.toURL(path);
return Channels.newChannel(warcUrl.openStream());
}
org.apache.hadoop.fs.Path hdfsPath = new org.apache.hadoop.fs.Path(path);