You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by rz...@apache.org on 2026/02/22 18:52:16 UTC
(stormcrawler) branch 1597 updated (6b0cd735 -> e9d0e404)
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a change to branch 1597
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
discard 6b0cd735 First steps in migration from URL to URI. Leads to some corner cases to discuss in normalizer / architecture of normalization.
add 2695a7df Bump aws.version from 1.12.795 to 1.12.796
add d44df67f Bump testcontainers.version from 1.21.3 to 1.21.4
add 9b4882e8 Minor: Regenerated License File for d44df67f7376ad398525df142fe0c4ecdcf0b112
add 2a6930af #1542 - Migrate Documentation from Wiki to Living Documentation in Code (#1714)
add 47a29da2 Fix #1763 - Documentation fixes from #1714 (#1765)
add a77acdab Fix #1761 - Add docker compose config to archetypes (#1764)
add 1c858556 Bump langchain4j.version from 1.9.1 to 1.10.0 (#1768)
add 23bc5113 Bump org.netpreserve:jwarc from 0.32.0 to 0.33.0 (#1767)
add 2adaa84e Minor: Regenerated License File for 23bc5113f566d7fc120ab0aa89da92b167790f56 (#1769)
add 64028dc4 Bump org.jsoup:jsoup from 1.21.2 to 1.22.1 (#1773)
add ba5d39f8 Bump aws.version from 1.12.796 to 1.12.797 (#1772)
add f082cf80 Bump org.codehaus.mojo:license-maven-plugin from 2.7.0 to 2.7.1 (#1771)
add 4fffc48f Minor: Regenerated License File for f082cf804cf9992df70a78b8d58f643c0f82c685 (#1774)
add e222c7d0 #1611 - Refactor SQL module to use PreparedStatement in SQLSpout and IndexerBolt (#1766)
add 190aa3cd Minor: Regenerated License File for e222c7d04cf3255c119fd1a5e80510c77a39d73d (#1777)
add 856d72da JDBC driver should be optional and not supplied to be vendor agnostic. (#1783)
add cfc37b79 Bump junit.version from 6.0.1 to 6.0.2 (#1779)
add f90345ac Bump com.puppycrawl.tools:checkstyle from 12.3.0 to 12.3.1 (#1780)
add 342fa11b Bump com.ibm.icu:icu4j from 78.1 to 78.2 (#1781)
add b12b7f53 Bump com.mysql:mysql-connector-j from 9.3.0 to 9.5.0 (#1778)
add df19b0c5 Minor: Regenerated License File for b12b7f53eaadee090c0c3810e532020c6e88f05d (#1784)
add 632b4960 [maven-release-plugin] prepare release stormcrawler-3.5.1
add 95e7520e [maven-release-plugin] prepare for next development iteration
add f56c4821 Bump actions/checkout from 6.0.1 to 6.0.2
add ff555b0d Bump peter-evans/create-pull-request from 8.0.0 to 8.1.0
add d004d2e1 Bump actions/setup-java from 5.1.0 to 5.2.0
add 8be76c74 Bump org.netpreserve:jwarc from 0.33.0 to 0.34.0
add d12784d3 Bump selenium.version from 4.39.0 to 4.40.0
add 0dce0f46 Bump actions/cache from 5.0.1 to 5.0.2
add dad0bdf7 Minor: Regenerated License File for d12784d3f583bd002f5aeb5f54892901a62b4f9f
add 38113645 Bump org.apache:apache from 35 to 37
add 0e401df3 fix: enable java 17 parsing for javadoc
add ddf076d1 #1775 add indentation rules, fix inconsistent files (#1776)
add 7e9505b7 link the docs instead of the obsolete wiki (#1795)
add 2c861ace Bump org.apache.solr:solr-solrj from 9.10.0 to 9.10.1 (#1796)
add 3baaf67a Minor: Regenerated License File for 2c861acef9e44e632999d16082fe69d26266fe98 (#1797)
add 1973ad79 Bump actions/cache from 5.0.2 to 5.0.3 (#1798)
add b6e77d73 Bump org.apache.maven.plugins:maven-compiler-plugin (#1800)
add 77dbbd47 Bump commons-codec:commons-codec from 1.20.0 to 1.21.0 (#1799)
add 9eef525e Bump com.microsoft.playwright:playwright from 1.57.0 to 1.58.0 (#1801)
add 1ad6f249 Bump com.mysql:mysql-connector-j from 9.5.0 to 9.6.0 (#1802)
add 55083925 Minor: Regenerated License File for 1ad6f249b6ff65b3eb5acfe73ce3c2ab04b29daf
add aa9f40ae Update Logo (#1805)
add 32eddd7a Bump langchain4j.version from 1.10.0 to 1.11.0
add 0d7fb580 Minor: Regenerated License File for 32eddd7a560e0b85da1b4e7714cbede4fc8d2d99
add c5337876 Bump junit.version from 6.0.2 to 6.0.3 (#1809)
add 96f3fb7d fix: use the local (runner) compose binary instead of a container in tests (#1811)
new e9d0e404 First steps in migration from URL to URI. Leads to some corner cases to discuss in normalizer / architecture of normalization.
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (6b0cd735)
\
N -- N -- N refs/heads/1597 (e9d0e404)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.asf.yaml | 3 +-
.editorconfig | 24 +
.github/workflows/main.yml | 10 +-
.github/workflows/maven.yml | 12 +-
.github/workflows/snapshots.yaml | 8 +-
.mvn/jvm.config | 2 +-
DISCLAIMER-BINARIES.txt | 9 +-
README.md | 4 +-
THIRD-PARTY.txt | 86 +-
archetype/pom.xml | 100 +-
.../META-INF/maven/archetype-metadata.xml | 68 +-
.../main/resources/archetype-resources/README.md | 8 +-
.../archetype-resources/crawler-conf.yaml | 18 +-
.../resources/archetype-resources/crawler.flux | 4 +-
.../archetype-resources/docker-compose.yml | 54 +
.../src/main/resources/archetype-resources/pom.xml | 252 ++--
.../main/resources/default-regex-normalizers.xml | 2 +-
assembly.xml | 128 +--
checkstyle.xml | 2 +-
core/pom.xml | 504 ++++----
core/src/main/resources/crawler-default.yaml | 36 +-
.../java/org/apache/stormcrawler/TestUtil.java | 38 +
core/src/test/resources/default-regex-filters.txt | 2 +-
core/src/test/resources/delegator-conf.yaml | 2 +-
core/src/test/resources/fast.urlfilter.json | 2 +-
core/src/test/resources/redir.html | 8 +-
.../stormcrawler.sitemap.extensions.all.xml | 154 +--
.../stormcrawler.sitemap.extensions.image.xml | 86 +-
.../stormcrawler.sitemap.extensions.links.xml | 62 +-
.../stormcrawler.sitemap.extensions.mobile.xml | 62 +-
.../stormcrawler.sitemap.extensions.news.xml | 84 +-
.../stormcrawler.sitemap.extensions.video.xml | 104 +-
.../test/resources/stormcrawler.sitemap.index.xml | 32 +-
core/src/test/resources/stormcrawler.sitemap.xml | 62 +-
core/src/test/resources/test.jsoupfilters.json | 2 +-
docs/pom.xml | 81 ++
.../tika-config.xml => docs/src/assembly/docs.xml | 36 +-
docs/src/main/asciidoc/architecture.adoc | 97 ++
docs/src/main/asciidoc/configuration.adoc | 316 +++++
docs/src/main/asciidoc/debugging.adoc | 21 +
docs/src/main/asciidoc/images/stormcrawler.drawio | 238 ++++
.../main/asciidoc/images/stormcrawler.drawio.jpg | Bin 0 -> 172493 bytes
.../main/asciidoc/images/stormcrawler.drawio.pdf | Bin 0 -> 50189 bytes
docs/src/main/asciidoc/index.adoc | 33 +
docs/src/main/asciidoc/internals.adoc | 443 +++++++
docs/src/main/asciidoc/overview.adoc | 53 +
docs/src/main/asciidoc/powered-by.adoc | 40 +
docs/src/main/asciidoc/presentations.adoc | 28 +
docs/src/main/asciidoc/quick-start.adoc | 211 ++++
external/ai/README.md | 2 +-
external/ai/ai-conf.yaml | 2 +-
external/ai/pom.xml | 4 +-
external/ai/src/main/resources/NOTICE | 2 +-
.../ai/src/main/resources/llm-default-prompt.txt | 2 +-
external/aws/pom.xml | 66 +-
external/langid/pom.xml | 46 +-
external/opensearch/archetype/pom.xml | 86 +-
.../META-INF/maven/archetype-metadata.xml | 94 +-
.../main/resources/archetype-resources/README.md | 14 +
.../archetype-resources/crawler-conf.yaml | 20 +-
.../resources/archetype-resources/crawler.flux | 2 +-
.../archetype-resources/dashboards/storm.ndjson | 2 +-
.../archetype-resources/docker-compose.yml | 81 ++
.../archetype-resources/opensearch-conf.yaml | 20 +-
.../src/main/resources/archetype-resources/pom.xml | 242 ++--
.../main/resources/default-regex-normalizers.xml | 2 +-
.../src/main/resources/indexer.mapping | 2 +-
.../src/main/resources/metrics.mapping | 2 +-
external/opensearch/opensearch-conf.yaml | 20 +-
external/opensearch/pom.xml | 184 +--
.../opensearch/src/test/resources/indexer.mapping | 2 +-
.../opensearch/src/test/resources/metrics.mapping | 2 +-
external/playwright/playwright-conf.yaml | 2 +-
external/playwright/pom.xml | 102 +-
external/pom.xml | 70 +-
external/selenium/pom.xml | 168 +--
external/solr/archetype/pom.xml | 2 +-
.../archetype-resources/clear-collections.sh | 2 +-
.../archetype-resources/crawler-conf.yaml | 20 +-
.../src/main/resources/archetype-resources/pom.xml | 242 ++--
.../main/resources/default-regex-normalizers.xml | 2 +-
external/solr/configsets/docs/conf/schema.xml | 42 +-
external/solr/configsets/docs/conf/solrconfig.xml | 40 +-
external/solr/configsets/metrics/conf/schema.xml | 26 +-
.../solr/configsets/metrics/conf/solrconfig.xml | 60 +-
external/solr/configsets/status/conf/schema.xml | 28 +-
.../solr/configsets/status/conf/solrconfig.xml | 38 +-
external/solr/pom.xml | 80 +-
.../stormcrawler/solr/SolrCloudContainerTest.java | 3 +-
external/sql/README.md | 10 +-
external/sql/pom.xml | 68 +-
external/sql/sql-conf.yaml | 14 +-
.../org/apache/stormcrawler/sql/IndexerBolt.java | 101 +-
.../java/org/apache/stormcrawler/sql/SQLSpout.java | 199 ++--
.../java/org/apache/stormcrawler/sql/SQLUtil.java | 21 +-
.../apache/stormcrawler/sql/StatusUpdaterBolt.java | 133 ++-
.../apache/stormcrawler/sql/AbstractSQLTest.java | 92 ++
.../apache/stormcrawler/sql/IndexerBoltTest.java | 311 +++++
.../org/apache/stormcrawler/sql/SQLSpoutTest.java | 237 ++++
.../stormcrawler/sql/StatusUpdaterBoltTest.java | 146 +++
external/tika/pom.xml | 102 +-
external/urlfrontier/pom.xml | 144 +--
external/warc/pom.xml | 138 +--
pom.xml | 1205 ++++++++++----------
104 files changed, 5551 insertions(+), 2827 deletions(-)
create mode 100644 .editorconfig
create mode 100644 archetype/src/main/resources/archetype-resources/docker-compose.yml
create mode 100644 docs/pom.xml
copy external/tika/src/main/resources/tika-config.xml => docs/src/assembly/docs.xml (50%)
create mode 100644 docs/src/main/asciidoc/architecture.adoc
create mode 100644 docs/src/main/asciidoc/configuration.adoc
create mode 100644 docs/src/main/asciidoc/debugging.adoc
create mode 100644 docs/src/main/asciidoc/images/stormcrawler.drawio
create mode 100644 docs/src/main/asciidoc/images/stormcrawler.drawio.jpg
create mode 100644 docs/src/main/asciidoc/images/stormcrawler.drawio.pdf
create mode 100644 docs/src/main/asciidoc/index.adoc
create mode 100644 docs/src/main/asciidoc/internals.adoc
create mode 100644 docs/src/main/asciidoc/overview.adoc
create mode 100644 docs/src/main/asciidoc/powered-by.adoc
create mode 100644 docs/src/main/asciidoc/presentations.adoc
create mode 100644 docs/src/main/asciidoc/quick-start.adoc
create mode 100644 external/opensearch/archetype/src/main/resources/archetype-resources/docker-compose.yml
create mode 100644 external/sql/src/test/java/org/apache/stormcrawler/sql/AbstractSQLTest.java
create mode 100644 external/sql/src/test/java/org/apache/stormcrawler/sql/IndexerBoltTest.java
create mode 100644 external/sql/src/test/java/org/apache/stormcrawler/sql/SQLSpoutTest.java
create mode 100644 external/sql/src/test/java/org/apache/stormcrawler/sql/StatusUpdaterBoltTest.java
(stormcrawler) 01/01: First steps in migration from URL to URI. Leads to some corner cases to discuss in normalizer / architecture of normalization.
Posted by rz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch 1597
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
commit e9d0e404a5a646634af4a49b42e6684a50973782
Author: Richard Zowalla <rz...@apache.org>
AuthorDate: Tue Dec 23 10:45:23 2025 +0100
First steps in migration from URL to URI. Leads to some corner cases to discuss in normalizer / architecture of normalization.
---
.../apache/stormcrawler/bolt/FeedParserBolt.java | 3 +-
.../org/apache/stormcrawler/bolt/FetcherBolt.java | 7 +-
.../apache/stormcrawler/bolt/JSoupParserBolt.java | 16 +++--
.../stormcrawler/bolt/SimpleFetcherBolt.java | 5 +-
.../stormcrawler/bolt/SiteMapParserBolt.java | 6 +-
.../stormcrawler/bolt/URLPartitionerBolt.java | 5 +-
.../apache/stormcrawler/filtering/URLFilters.java | 3 +-
.../filtering/basic/BasicURLNormalizer.java | 22 ++++--
.../stormcrawler/filtering/host/HostURLFilter.java | 6 +-
.../filtering/regex/FastURLFilter.java | 8 ++-
.../filtering/robots/RobotsFilter.java | 6 +-
.../stormcrawler/indexing/AbstractIndexerBolt.java | 6 +-
.../apache/stormcrawler/jsoup/LinkParseFilter.java | 8 ++-
.../apache/stormcrawler/parse/JSoupFilters.java | 6 +-
.../apache/stormcrawler/parse/ParseFilters.java | 6 +-
.../stormcrawler/parse/filter/LinkParseFilter.java | 8 ++-
.../stormcrawler/protocol/RobotRulesParser.java | 4 +-
.../stormcrawler/protocol/file/FileResponse.java | 6 +-
.../stormcrawler/protocol/okhttp/HttpProtocol.java | 6 +-
.../apache/stormcrawler/util/URLPartitioner.java | 6 +-
.../java/org/apache/stormcrawler/util/URLUtil.java | 17 ++---
.../stormcrawler/filtering/BasicURLFilterTest.java | 14 ++--
.../filtering/BasicURLNormalizerTest.java | 84 +++++++++++-----------
.../stormcrawler/filtering/FastURLFilterTest.java | 16 +++--
.../stormcrawler/filtering/HostURLFilterTest.java | 18 ++---
.../stormcrawler/filtering/MaxDepthFilterTest.java | 18 ++---
.../filtering/MetadataFilterFromJsonTest.java | 38 +++++-----
.../stormcrawler/filtering/MetadataFilterTest.java | 58 +++++++--------
.../stormcrawler/filtering/RegexFilterTest.java | 20 +++---
.../stormcrawler/util/CookieConverterTest.java | 6 +-
.../protocol/selenium/RemoteDriverProtocol.java | 3 +-
.../org/apache/stormcrawler/tika/ParserBolt.java | 6 +-
.../org/apache/stormcrawler/warc/WARCSpout.java | 6 +-
33 files changed, 257 insertions(+), 190 deletions(-)
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java
index 3abe00ff..7c451167 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java
@@ -25,6 +25,7 @@ import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.io.SyndFeedInput;
import java.io.ByteArrayInputStream;
+import java.net.URI;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
@@ -175,7 +176,7 @@ public class FeedParserBolt extends StatusEmitterBolt {
feed = input.build(new InputSource(is));
}
- URL url1 = new URL(url);
+ URL url1 = new URI(url).toURL();
List<SyndEntry> entries = feed.getEntries();
for (SyndEntry entry : entries) {
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
index 295631b1..c07474a2 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
@@ -22,6 +22,7 @@ import crawlercommons.robots.BaseRobotRules;
import java.io.File;
import java.net.InetAddress;
import java.net.MalformedURLException;
+import java.net.URI;
import java.net.URL;
import java.net.UnknownHostException;
import java.time.Instant;
@@ -529,7 +530,7 @@ public class FetcherBolt extends StatusEmitterBolt {
boolean asap = false;
try {
- URL url = new URL(fit.url);
+ URL url = new URI(fit.url).toURL();
Protocol protocol = protocolFactory.getProtocol(url);
if (protocol == null) {
@@ -982,8 +983,8 @@ public class FetcherBolt extends StatusEmitterBolt {
URL url;
try {
- url = new URL(urlString);
- } catch (MalformedURLException e) {
+ url = new URI(urlString).toURL();
+ } catch (Exception e) {
LOG.error("{} is a malformed URL", urlString);
Metadata metadata = (Metadata) input.getValueByField("metadata");
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
index 933f41bd..9970bb8f 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
@@ -24,6 +24,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
@@ -293,7 +295,7 @@ public class JSoupParserBolt extends StatusEmitterBolt {
} else {
final Elements links = jsoupDoc.select("a[href]");
slinks = new HashMap<>(links.size());
- final URL baseUrl = new URL(url);
+ final URL baseUrl = new URI(url).toURL();
for (Element link : links) {
// nofollow
String[] relkeywords = link.attr("rel").split(" ");
@@ -375,7 +377,7 @@ public class JSoupParserBolt extends StatusEmitterBolt {
// https://github.com/apache/stormcrawler/issues/954
if (allowRedirs() && StringUtils.isNotBlank(redirection)) {
- emitOutlink(tuple, new URL(url), redirection, metadata);
+ emitOutlink(tuple, new URI(url).toURL(), redirection, metadata);
}
// Mark URL as redirected
@@ -387,8 +389,8 @@ public class JSoupParserBolt extends StatusEmitterBolt {
eventCounter.scope("tuple_success").incr();
return;
}
- } catch (MalformedURLException e) {
- LOG.error("MalformedURLException on {}", url);
+ } catch (MalformedURLException | URISyntaxException e) {
+ LOG.error("Exception on {}", url, e);
}
}
@@ -516,11 +518,11 @@ public class JSoupParserBolt extends StatusEmitterBolt {
URL sourceUrl;
try {
- sourceUrl = new URL(url);
- } catch (MalformedURLException e) {
+ sourceUrl = new URI(url).toURL();
+ } catch (Exception e) {
// we would have known by now as previous components check whether
// the URL is valid
- LOG.error("MalformedURLException on {}", url);
+ LOG.error("Exception on {}", url, e);
eventCounter.scope("error_invalid_source_url").incrBy(1);
return new LinkedList<>();
}
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
index e5eb16f5..c79d7449 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
@@ -23,6 +23,7 @@ import crawlercommons.domains.PaidLevelDomain;
import crawlercommons.robots.BaseRobotRules;
import java.net.InetAddress;
import java.net.MalformedURLException;
+import java.net.URI;
import java.net.URL;
import java.net.UnknownHostException;
import java.text.SimpleDateFormat;
@@ -265,8 +266,8 @@ public class SimpleFetcherBolt extends StatusEmitterBolt {
URL url;
try {
- url = new URL(urlString);
- } catch (MalformedURLException e) {
+ url = new URI(urlString).toURL();
+ } catch (Exception e) {
LOG.error("{} is a malformed URL", urlString);
// Report to status stream and ack
metadata.setValue(Constants.STATUS_ERROR_CAUSE, "malformed URL");
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java
index 6736b41f..acbebf41 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java
@@ -31,6 +31,8 @@ import crawlercommons.sitemaps.UnknownFormatException;
import crawlercommons.sitemaps.extension.Extension;
import crawlercommons.sitemaps.extension.ExtensionMetadata;
import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
@@ -181,9 +183,9 @@ public class SiteMapParserBolt extends StatusEmitterBolt {
private List<Outlink> parseSiteMap(
String url, byte[] content, String contentType, Metadata parentMetadata)
- throws UnknownFormatException, IOException {
+ throws UnknownFormatException, IOException, URISyntaxException {
- URL url1 = new URL(url);
+ URL url1 = new URI(url).toURL();
long start = System.currentTimeMillis();
AbstractSiteMap siteMap;
// let the parser guess what the mimetype is
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/URLPartitionerBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/URLPartitionerBolt.java
index 8f44779f..f11f2a8e 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/URLPartitionerBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/URLPartitionerBolt.java
@@ -20,6 +20,7 @@ package org.apache.stormcrawler.bolt;
import crawlercommons.domains.PaidLevelDomain;
import java.net.InetAddress;
import java.net.MalformedURLException;
+import java.net.URI;
import java.net.URL;
import java.util.Collections;
import java.util.LinkedHashMap;
@@ -82,9 +83,9 @@ public class URLPartitionerBolt extends BaseRichBolt {
if (partitionKey == null) {
URL u;
try {
- u = new URL(url);
+ u = new URI(url).toURL();
host = u.getHost();
- } catch (MalformedURLException e1) {
+ } catch (Exception e1) {
eventCounter.scope("Invalid URL").incrBy(1);
LOG.warn("Invalid URL: {}", url);
// ack it so that it doesn't get replayed
diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/URLFilters.java b/core/src/main/java/org/apache/stormcrawler/filtering/URLFilters.java
index 59f36b6f..ce85ec3e 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/URLFilters.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/URLFilters.java
@@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.io.InputStream;
+import java.net.URI;
import java.net.URL;
import java.util.List;
import java.util.Map;
@@ -185,7 +186,7 @@ public class URLFilters extends URLFilter implements JSONResource {
for (URLFilter filter : filters.filters) {
long start = System.currentTimeMillis();
normalizedUrl =
- filter.filter(new URL(sourceUrl), new Metadata(), normalizedUrl);
+ filter.filter(new URI(sourceUrl).toURL(), new Metadata(), normalizedUrl);
long end = System.currentTimeMillis();
System.out.println(
"\t["
diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java
index 9a5692ef..ca029223 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java
@@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode;
import java.net.IDN;
import java.net.MalformedURLException;
import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
@@ -120,7 +121,7 @@ public class BasicURLNormalizer extends URLFilter {
}
try {
- URL theUrl = new URL(urlToFilter);
+ URL theUrl = new URI(urlToFilter).toURL();
String file = theUrl.getFile();
String protocol = theUrl.getProtocol();
String host = theUrl.getHost();
@@ -152,9 +153,18 @@ public class BasicURLNormalizer extends URLFilter {
hasChanged = true;
}
if (hasChanged) {
- urlToFilter = new URL(protocol, host, port, file2).toString();
+ URI uri = new URI(
+ protocol,
+ null, // userInfo
+ host,
+ port,
+ file2, // path
+ null, // query
+ null // fragment
+ );
+ urlToFilter = uri.toString();
}
- } catch (MalformedURLException e) {
+ } catch (MalformedURLException | URISyntaxException e) {
return null;
}
@@ -223,7 +233,7 @@ public class BasicURLNormalizer extends URLFilter {
try {
// Handle illegal characters by making a url first
// this will clean illegal characters like |
- final URL url = new URL(urlToFilter);
+ final URL url = new URI(urlToFilter).toURL();
String query = url.getQuery();
String path = url.getPath();
@@ -287,8 +297,8 @@ public class BasicURLNormalizer extends URLFilter {
+ ((s = newQueryString) != null ? '?' + s : "")
+ ((s = url.getRef()) != null ? '#' + s : "");
- } catch (MalformedURLException e) {
- LOG.warn("Invalid urlToFilter {}. {}", urlToFilter, e);
+ } catch (MalformedURLException | URISyntaxException e) {
+ LOG.warn("Invalid urlToFilter {}.", urlToFilter, e);
return null;
}
}
diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java
index f7e5055b..ab104a66 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java
@@ -20,6 +20,8 @@ package org.apache.stormcrawler.filtering.host;
import com.fasterxml.jackson.databind.JsonNode;
import crawlercommons.domains.PaidLevelDomain;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.Map;
import org.apache.stormcrawler.Metadata;
@@ -82,8 +84,8 @@ public class HostURLFilter extends URLFilter {
URL url;
try {
- url = new URL(urlToFilter);
- } catch (MalformedURLException e1) {
+ url = new URI(urlToFilter).toURL();
+ } catch (MalformedURLException | URISyntaxException e1) {
return null;
}
diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/regex/FastURLFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/regex/FastURLFilter.java
index 854b464a..9f2c9b3a 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/regex/FastURLFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/regex/FastURLFilter.java
@@ -24,6 +24,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
@@ -177,7 +179,7 @@ public class FastURLFilter extends URLFilter implements JSONResource {
if (rules.filter(urlToFilter, sourceMetadata)) {
return null;
}
- } catch (MalformedURLException e) {
+ } catch (MalformedURLException | URISyntaxException e) {
return null;
}
return urlToFilter;
@@ -210,8 +212,8 @@ class Rules {
*
* @throws MalformedURLException
*/
- public boolean filter(String url, Metadata metadata) throws MalformedURLException {
- URL u = new URL(url);
+ public boolean filter(String url, Metadata metadata) throws MalformedURLException, URISyntaxException {
+ URL u = new URI(url).toURL();
// first try the full hostname
String hostname = u.getHost();
diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/robots/RobotsFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/robots/RobotsFilter.java
index 07140c2f..42e775db 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/robots/RobotsFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/robots/RobotsFilter.java
@@ -20,6 +20,8 @@ package org.apache.stormcrawler.filtering.robots;
import com.fasterxml.jackson.databind.JsonNode;
import crawlercommons.robots.BaseRobotRules;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.Map;
import org.apache.storm.Config;
@@ -61,8 +63,8 @@ public class RobotsFilter extends URLFilter {
@NotNull String urlToFilter) {
URL target;
try {
- target = new URL(urlToFilter);
- } catch (MalformedURLException e) {
+ target = new URI(urlToFilter).toURL();
+ } catch (MalformedURLException | URISyntaxException e) {
return null;
}
diff --git a/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java b/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java
index 91175846..4ece0b75 100644
--- a/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java
@@ -19,6 +19,8 @@ package org.apache.stormcrawler.indexing;
import crawlercommons.domains.PaidLevelDomain;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
@@ -286,7 +288,7 @@ public abstract class AbstractIndexerBolt extends BaseRichBolt {
}
try {
- URL url1 = new URL(url);
+ URL url1 = new URI(url).toURL();
URL canonical = URLUtil.resolveUrl(url1, canonicalValue);
String domain = PaidLevelDomain.getPLD(url1.getHost());
@@ -298,7 +300,7 @@ public abstract class AbstractIndexerBolt extends BaseRichBolt {
} else {
log.info("Canonical URL references a different domain, ignoring in {} ", url);
}
- } catch (MalformedURLException e) {
+ } catch (MalformedURLException | URISyntaxException e) {
log.error("Malformed canonical URL {} was found in {} ", canonicalValue, url);
}
diff --git a/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java b/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java
index 1125c374..87d66751 100644
--- a/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/jsoup/LinkParseFilter.java
@@ -19,6 +19,8 @@ package org.apache.stormcrawler.jsoup;
import com.fasterxml.jackson.databind.JsonNode;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
@@ -73,11 +75,11 @@ public class LinkParseFilter extends XPathFilter {
java.net.URL sourceUrl;
try {
- sourceUrl = new URL(url);
- } catch (MalformedURLException e1) {
+ sourceUrl = new URI(url).toURL();
+ } catch (MalformedURLException | URISyntaxException e1) {
// we would have known by now as previous components check whether
// the URL is valid
- LOG.error("MalformedURLException on {}", url);
+ LOG.error("Malformed URL on {}", url);
return;
}
diff --git a/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java b/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java
index 8fd8adc3..9a9ccb36 100644
--- a/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java
@@ -21,6 +21,8 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.List;
@@ -126,7 +128,7 @@ public class JSoupFilters extends AbstractConfigurable implements JSoupFilter, J
}
/** * Used for quick testing + debugging */
- public static void main(String[] args) throws IOException, ParseException {
+ public static void main(String[] args) throws IOException, ParseException, URISyntaxException {
Config conf = new Config();
@@ -154,7 +156,7 @@ public class JSoupFilters extends AbstractConfigurable implements JSoupFilter, J
String url = cmd.getArgs()[0];
- byte[] content = IOUtils.toByteArray((new URL(url)).openStream());
+ byte[] content = IOUtils.toByteArray((new URI(url).toURL()).openStream());
Document doc = Jsoup.parse(new String(content, StandardCharsets.UTF_8), url);
diff --git a/core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java b/core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java
index 6852d590..9f2ae827 100644
--- a/core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java
@@ -23,6 +23,8 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.List;
@@ -159,7 +161,7 @@ public class ParseFilters extends ParseFilter implements JSONResource {
*
* @since 1.17
*/
- public static void main(String[] args) throws IOException, ParseException {
+ public static void main(String[] args) throws IOException, ParseException, URISyntaxException {
Config conf = new Config();
@@ -187,7 +189,7 @@ public class ParseFilters extends ParseFilter implements JSONResource {
String url = cmd.getArgs()[0];
- byte[] content = IOUtils.toByteArray((new URL(url)).openStream());
+ byte[] content = IOUtils.toByteArray((new URI(url).toURL()).openStream());
Document doc = Jsoup.parse(new String(content, StandardCharsets.UTF_8), url);
diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java
index d379e014..eb23c795 100644
--- a/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/LinkParseFilter.java
@@ -19,6 +19,8 @@ package org.apache.stormcrawler.parse.filter;
import com.fasterxml.jackson.databind.JsonNode;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
@@ -73,11 +75,11 @@ public class LinkParseFilter extends XPathFilter {
java.net.URL sourceUrl;
try {
- sourceUrl = new URL(url);
- } catch (MalformedURLException e1) {
+ sourceUrl = new URI(url).toURL();
+ } catch (MalformedURLException | URISyntaxException e1) {
// we would have known by now as previous components check whether
// the URL is valid
- LOG.error("MalformedURLException on {}", url);
+ LOG.error("Malformed URL on {}", url);
return;
}
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java
index 63849eec..00198119 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java
@@ -23,6 +23,8 @@ import crawlercommons.robots.BaseRobotRules;
import crawlercommons.robots.SimpleRobotRules;
import crawlercommons.robots.SimpleRobotRules.RobotRulesMode;
import crawlercommons.robots.SimpleRobotRulesParser;
+
+import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
@@ -200,7 +202,7 @@ public abstract class RobotRulesParser {
public BaseRobotRules getRobotRulesSet(Protocol protocol, String url) {
URL u;
try {
- u = new URL(url);
+ u = new URI(url).toURL();
} catch (Exception e) {
return EMPTY_RULES;
}
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java
index e5858f35..88d1de16 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java
@@ -21,6 +21,8 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
@@ -45,13 +47,13 @@ public class FileResponse {
private int statusCode;
private final Metadata metadata;
- public FileResponse(String u, Metadata md, FileProtocol fileProtocol) throws IOException {
+ public FileResponse(String u, Metadata md, FileProtocol fileProtocol) throws IOException, URISyntaxException {
metadata = new Metadata();
content = new byte[0];
statusCode = HttpStatus.SC_INTERNAL_SERVER_ERROR;
- URL url = new URL(u);
+ URL url = new URI(u).toURL();
if (!url.getPath().equals(url.getFile())) {
LOG.warn("url.getPath() != url.getFile(): {}.", url);
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
index e8a14eb9..89d99934 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
@@ -21,6 +21,8 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.Proxy;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.security.cert.CertificateException;
@@ -271,11 +273,11 @@ public class HttpProtocol extends AbstractHttpProtocol {
return;
}
try {
- final List<Cookie> cookies = CookieConverter.getCookies(cookieStrings, new URL(url));
+ final List<Cookie> cookies = CookieConverter.getCookies(cookieStrings, new URI(url).toURL());
for (Cookie c : cookies) {
rb.addHeader("Cookie", c.getName() + "=" + c.getValue());
}
- } catch (MalformedURLException e) { // Bad url , nothing to do
+ } catch (MalformedURLException | URISyntaxException e) { // Bad url , nothing to do
}
}
diff --git a/core/src/main/java/org/apache/stormcrawler/util/URLPartitioner.java b/core/src/main/java/org/apache/stormcrawler/util/URLPartitioner.java
index 384c3eeb..b8f11ec0 100644
--- a/core/src/main/java/org/apache/stormcrawler/util/URLPartitioner.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/URLPartitioner.java
@@ -20,6 +20,8 @@ package org.apache.stormcrawler.util;
import crawlercommons.domains.PaidLevelDomain;
import java.net.InetAddress;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
@@ -58,9 +60,9 @@ public class URLPartitioner {
if (partitionKey == null) {
URL u;
try {
- u = new URL(url);
+ u = new URI(url).toURL();
host = u.getHost();
- } catch (MalformedURLException e1) {
+ } catch (MalformedURLException | URISyntaxException e) {
LOG.warn("Invalid URL: {}", url);
return null;
}
diff --git a/core/src/main/java/org/apache/stormcrawler/util/URLUtil.java b/core/src/main/java/org/apache/stormcrawler/util/URLUtil.java
index fe3c72f2..1350ece0 100644
--- a/core/src/main/java/org/apache/stormcrawler/util/URLUtil.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/URLUtil.java
@@ -20,6 +20,7 @@ package org.apache.stormcrawler.util;
import java.net.IDN;
import java.net.MalformedURLException;
import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.Locale;
import java.util.regex.Pattern;
@@ -137,8 +138,8 @@ public class URLUtil {
*
* @throws MalformedURLException
*/
- public static String[] getHostSegments(String url) throws MalformedURLException {
- return getHostSegments(new URL(url));
+ public static String[] getHostSegments(String url) throws MalformedURLException, URISyntaxException {
+ return getHostSegments(new URI(url).toURL());
}
/**
@@ -149,8 +150,8 @@ public class URLUtil {
*/
public static String getHost(String url) {
try {
- return new URL(url).getHost().toLowerCase(Locale.ROOT);
- } catch (MalformedURLException e) {
+ return new URI(url).toURL().getHost().toLowerCase(Locale.ROOT);
+ } catch (MalformedURLException | URISyntaxException e) {
return null;
}
}
@@ -167,16 +168,16 @@ public class URLUtil {
// get the full url, and replace the query string with and empty
// string
url = url.toLowerCase(Locale.ROOT);
- String queryStr = new URL(url).getQuery();
+ String queryStr = new URI(url).toURL().getQuery();
return (queryStr != null) ? url.replace("?" + queryStr, "") : url;
- } catch (MalformedURLException e) {
+ } catch (MalformedURLException | URISyntaxException e) {
return null;
}
}
public static String toASCII(String url) {
try {
- URL u = new URL(url);
+ URL u = new URI(url).toURL();
URI p =
new URI(
u.getProtocol(),
@@ -195,7 +196,7 @@ public class URLUtil {
public static String toUNICODE(String url) {
try {
- URL u = new URL(url);
+ URL u = new URI(url).toURL();
URI p =
new URI(
u.getProtocol(),
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLFilterTest.java
index 017b60b8..e59f8cc4 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLFilterTest.java
@@ -19,6 +19,8 @@ package org.apache.stormcrawler.filtering;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
@@ -40,25 +42,25 @@ class BasicURLFilterTest {
}
@Test
- void testRepetition() throws MalformedURLException {
+ void testRepetition() throws MalformedURLException, URISyntaxException {
URLFilter filter = createFilter(-1, 3);
Metadata metadata = new Metadata();
- URL targetURL = new URL("http://www.sourcedomain.com/a/a/a/index.html");
+ URL targetURL = new URI("http://www.sourcedomain.com/a/a/a/index.html").toURL();
String filterResult = filter.filter(targetURL, metadata, targetURL.toExternalForm());
Assertions.assertNull(filterResult);
- targetURL = new URL("http://www.sourcedomain.com/a/b/a/index.html");
+ targetURL = new URI("http://www.sourcedomain.com/a/b/a/index.html").toURL();
filterResult = filter.filter(targetURL, metadata, targetURL.toExternalForm());
Assertions.assertEquals(targetURL.toExternalForm(), filterResult);
}
@Test
- void testLength() throws MalformedURLException {
+ void testLength() throws MalformedURLException, URISyntaxException {
URLFilter filter = createFilter(32, -1);
Metadata metadata = new Metadata();
- URL targetURL = new URL("http://www.sourcedomain.com/a/a/a/index.html");
+ URL targetURL = new URI("http://www.sourcedomain.com/a/a/a/index.html").toURL();
String filterResult = filter.filter(targetURL, metadata, targetURL.toExternalForm());
Assertions.assertNull(filterResult);
- targetURL = new URL("http://www.sourcedomain.com/");
+ targetURL = new URI("http://www.sourcedomain.com/").toURL();
filterResult = filter.filter(targetURL, metadata, targetURL.toExternalForm());
Assertions.assertEquals(targetURL.toExternalForm(), filterResult);
}
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
index 65da7630..65676040 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java
@@ -23,6 +23,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.Arrays;
import java.util.HashMap;
@@ -79,9 +81,9 @@ class BasicURLNormalizerTest {
}
@Test
- void testAnchorFilter() throws MalformedURLException {
+ void testAnchorFilter() throws MalformedURLException, URISyntaxException {
URLFilter allAllowed = createFilter(true, false);
- URL url = new URL("http://www.sourcedomain.com/#0");
+ URL url = new URI("http://www.sourcedomain.com/#0").toURL();
Metadata metadata = new Metadata();
String filterResult = allAllowed.filter(url, metadata, url.toExternalForm());
String expected = "http://www.sourcedomain.com/";
@@ -89,18 +91,18 @@ class BasicURLNormalizerTest {
}
@Test
- void testAnchorFilterFalse() throws MalformedURLException {
+ void testAnchorFilterFalse() throws MalformedURLException, URISyntaxException {
URLFilter allAllowed = createFilter(false, false);
- URL url = new URL("http://www.sourcedomain.com/#0");
+ URL url = new URI("http://www.sourcedomain.com/#0").toURL();
Metadata metadata = new Metadata();
String filterResult = allAllowed.filter(url, metadata, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
}
@Test
- void testRemoveSomeOfManyQueryParams() throws MalformedURLException {
+ void testRemoveSomeOfManyQueryParams() throws MalformedURLException, URISyntaxException {
URLFilter urlFilter = createFilter(queryParamsToFilter);
- URL testSourceUrl = new URL("http://google.com");
+ URL testSourceUrl = new URI("http://google.com").toURL();
String testUrl = "http://google.com?keep1=true&a=c&foo=baz&keep2=true";
String expectedResult = "http://google.com?keep1=true&keep2=true";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -108,9 +110,9 @@ class BasicURLNormalizerTest {
}
@Test
- void testRemoveAllQueryParams() throws MalformedURLException {
+ void testRemoveAllQueryParams() throws MalformedURLException, URISyntaxException {
URLFilter urlFilter = createFilter(queryParamsToFilter);
- URL testSourceUrl = new URL("http://google.com");
+ URL testSourceUrl = new URI("http://google.com").toURL();
String testUrl = "http://google.com?a=c&foo=baz";
String expectedResult = "http://google.com";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -118,9 +120,9 @@ class BasicURLNormalizerTest {
}
@Test
- void testRemoveDupeQueryParams() throws MalformedURLException {
+ void testRemoveDupeQueryParams() throws MalformedURLException, URISyntaxException {
URLFilter urlFilter = createFilter(queryParamsToFilter);
- URL testSourceUrl = new URL("http://google.com");
+ URL testSourceUrl = new URI("http://google.com").toURL();
String testUrl = "http://google.com?a=c&foo=baz&foo=bar&test=true";
String expectedResult = "http://google.com?test=true";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -128,9 +130,9 @@ class BasicURLNormalizerTest {
}
@Test
- void testPipeInUrlAndFilterStillWorks() throws MalformedURLException {
+ void testPipeInUrlAndFilterStillWorks() throws MalformedURLException, URISyntaxException {
URLFilter urlFilter = createFilter(queryParamsToFilter);
- URL testSourceUrl = new URL("http://google.com");
+ URL testSourceUrl = new URI("http://google.com").toURL();
String testUrl = "http://google.com?a=c|d&foo=baz&foo=bar&test=true";
String expectedResult = "http://google.com?test=true";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -138,9 +140,9 @@ class BasicURLNormalizerTest {
}
@Test
- void testBothAnchorAndQueryFilter() throws MalformedURLException {
+ void testBothAnchorAndQueryFilter() throws MalformedURLException, URISyntaxException {
URLFilter urlFilter = createFilter(true, queryParamsToFilter);
- URL testSourceUrl = new URL("http://google.com");
+ URL testSourceUrl = new URI("http://google.com").toURL();
String testUrl = "http://google.com?a=c|d&foo=baz&foo=bar&test=true#fragment=ohYeah";
String expectedResult = "http://google.com?test=true";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -148,9 +150,9 @@ class BasicURLNormalizerTest {
}
@Test
- void testQuerySort() throws MalformedURLException {
+ void testQuerySort() throws MalformedURLException, URISyntaxException {
URLFilter urlFilter = createFilter(queryParamsToFilter);
- URL testSourceUrl = new URL("http://google.com");
+ URL testSourceUrl = new URI("http://google.com").toURL();
String testUrl = "http://google.com?a=c|d&foo=baz&foo=bar&test=true&z=2&d=4";
String expectedResult = "http://google.com?d=4&test=true&z=2";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -158,9 +160,9 @@ class BasicURLNormalizerTest {
}
@Test
- void testMangledQueryString() throws MalformedURLException {
+ void testMangledQueryString() throws MalformedURLException, URISyntaxException {
URLFilter urlFilter = createFilter(queryParamsToFilter);
- URL testSourceUrl = new URL("http://google.com");
+ URL testSourceUrl = new URI("http://google.com").toURL();
String testUrl = "http://google.com&d=4&good=true";
String expectedResult = "http://google.com?d=4&good=true";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -168,11 +170,11 @@ class BasicURLNormalizerTest {
}
@Test
- void testHashes() throws MalformedURLException {
+ void testHashes() throws MalformedURLException, URISyntaxException {
ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance);
filterParams.put("removeHashes", true);
URLFilter urlFilter = createFilter(filterParams);
- URL testSourceUrl = new URL("http://florida-chemical.com");
+ URL testSourceUrl = new URI("http://florida-chemical.com").toURL();
String in =
"http://www.florida-chemical.com/Diacetone-Alcohol-DAA-99.html?xid_0b629=12854b827878df26423d933a5baf86d5";
String out = "http://www.florida-chemical.com/Diacetone-Alcohol-DAA-99.html";
@@ -186,9 +188,9 @@ class BasicURLNormalizerTest {
}
@Test
- void testDontFixMangledQueryString() throws MalformedURLException {
+ void testDontFixMangledQueryString() throws MalformedURLException, URISyntaxException {
URLFilter urlFilter = createFilter(true, false, queryParamsToFilter);
- URL testSourceUrl = new URL("http://google.com");
+ URL testSourceUrl = new URI("http://google.com").toURL();
String testUrl = "http://google.com&d=4&good=true";
String expectedResult = "http://google.com&d=4&good=true";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -196,14 +198,14 @@ class BasicURLNormalizerTest {
}
@Test
- void testFixMangledQueryString() throws MalformedURLException {
+ void testFixMangledQueryString() throws MalformedURLException, URISyntaxException {
URLFilter urlFilter = createFilter(false, true, queryParamsToFilter);
- URL testSourceUrl = new URL("http://google.com");
+ URL testSourceUrl = new URI("http://google.com").toURL();
String testUrl = "http://google.com&d=4&good=true";
String expectedResult = "http://google.com?d=4&good=true";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
assertEquals(expectedResult, normalizedUrl, "Failed to filter query string");
- testSourceUrl = new URL("http://dev.com");
+ testSourceUrl = new URI("http://dev.com").toURL();
testUrl = "http://dev.com/s&utax/NEWSRLSEfy18.pdf";
normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
expectedResult = "http://dev.com/s&utax/NEWSRLSEfy18.pdf";
@@ -211,11 +213,11 @@ class BasicURLNormalizerTest {
}
@Test
- void testProperURLEncodingWithoutQueryParameter() throws MalformedURLException {
+ void testProperURLEncodingWithoutQueryParameter() throws MalformedURLException, URISyntaxException {
URLFilter urlFilter = createFilter(queryParamsToFilter);
String urlWithEscapedCharacters =
"http://www.dillards.com/product/ASICS-Womens-GT2000-3-LiteShow%E2%84%A2-Running-Shoes_301_-1_301_504736989";
- URL testSourceUrl = new URL(urlWithEscapedCharacters);
+ URL testSourceUrl = new URI(urlWithEscapedCharacters).toURL();
String testUrl = urlWithEscapedCharacters;
String expectedResult = urlWithEscapedCharacters;
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -223,11 +225,11 @@ class BasicURLNormalizerTest {
}
@Test
- void testProperURLEncodingWithQueryParameters() throws MalformedURLException {
+ void testProperURLEncodingWithQueryParameters() throws MalformedURLException, URISyntaxException {
URLFilter urlFilter = createFilter(queryParamsToFilter);
String urlWithEscapedCharacters =
"http://www.dillards.com/product/ASICS-Womens-GT2000-3-LiteShow%E2%84%A2-Running-Shoes_301_-1_301_504736989?how=are&you=doing";
- URL testSourceUrl = new URL(urlWithEscapedCharacters);
+ URL testSourceUrl = new URI(urlWithEscapedCharacters).toURL();
String testUrl = urlWithEscapedCharacters;
String expectedResult = urlWithEscapedCharacters;
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
@@ -235,24 +237,24 @@ class BasicURLNormalizerTest {
}
@Test
- void testProperURLEncodingWithBackSlash() throws MalformedURLException {
+ void testProperURLEncodingWithBackSlash() throws MalformedURLException, URISyntaxException {
URLFilter urlFilter = createFilter(queryParamsToFilter);
String urlWithEscapedCharacters =
"http://www.voltaix.com/\\SDS\\Silicon\\Trisilane\\Trisilane_SI050_USENG.pdf";
String expectedResult =
"http://www.voltaix.com/%5CSDS%5CSilicon%5CTrisilane%5CTrisilane_SI050_USENG.pdf";
- URL testSourceUrl = new URL(urlWithEscapedCharacters);
+ URL testSourceUrl = new URI(urlWithEscapedCharacters).toURL();
String testUrl = urlWithEscapedCharacters;
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), testUrl);
assertEquals(expectedResult, normalizedUrl, "Failed to filter query string");
}
@Test
- void testInvalidURI() throws MalformedURLException {
+ void testInvalidURI() throws MalformedURLException, URISyntaxException {
URLFilter urlFilter = createFilter(true, true);
// this one is now handled by the normaliser
String nonURI = "http://www.quanjing.com/search.aspx?q=top-651451||1|60|1|2||||&Fr=4";
- URL testSourceUrl = new URL(nonURI);
+ URL testSourceUrl = new URI(nonURI).toURL();
String expectedResult =
"http://www.quanjing.com/search.aspx?q=top-651451%7C%7C1%7C60%7C1%7C2%7C%7C%7C%7C&Fr=4";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), nonURI);
@@ -260,7 +262,7 @@ class BasicURLNormalizerTest {
// this one is
nonURI =
"http://vins.lemonde.fr?utm_source=LeMonde_partenaire_hp&utm_medium=EMPLACEMENT PARTENAIRE&utm_term=&utm_content=&utm_campaign=LeMonde_partenaire_hp";
- testSourceUrl = new URL(nonURI);
+ testSourceUrl = new URI(nonURI).toURL();
expectedResult =
"http://vins.lemonde.fr?utm_source=LeMonde_partenaire_hp&utm_medium=EMPLACEMENT%20PARTENAIRE&utm_term=&utm_content=&utm_campaign=LeMonde_partenaire_hp";
normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), nonURI);
@@ -269,16 +271,16 @@ class BasicURLNormalizerTest {
// http://docs.oracle.com/javase/7/docs/api/java/net/URI.html#normalize()
String nonNormURL =
"http://docs.oracle.com/javase/7/docs/api/java/net/../net/./URI.html#normalize()";
- testSourceUrl = new URL(nonNormURL);
+ testSourceUrl = new URI(nonNormURL).toURL();
expectedResult = "http://docs.oracle.com/javase/7/docs/api/java/net/URI.html";
normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), nonNormURL);
assertEquals(expectedResult, normalizedUrl, "Failed to filter query string");
}
@Test
- void testLowerCasing() throws MalformedURLException {
+ void testLowerCasing() throws MalformedURLException, URISyntaxException {
URLFilter urlFilter = createFilter(false, false);
- URL testSourceUrl = new URL("http://blablabla.org/");
+ URL testSourceUrl = new URI("http://blablabla.org/").toURL();
String inputURL = "HTTP://www.quanjing.com/";
String expectedResult = inputURL.toLowerCase(Locale.ROOT);
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), inputURL);
@@ -291,9 +293,9 @@ class BasicURLNormalizerTest {
// https://github.com/apache/stormcrawler/issues/401
@Test
- void testNonStandardPercentEncoding() throws MalformedURLException {
+ void testNonStandardPercentEncoding() throws MalformedURLException, URISyntaxException {
URLFilter urlFilter = createFilter(false, false);
- URL testSourceUrl = new URL("http://www.hurriyet.com.tr/index/?d=20160328&p=13");
+ URL testSourceUrl = new URI("http://www.hurriyet.com.tr/index/?d=20160328&p=13").toURL();
String inputURL = "http://www.hurriyet.com.tr/index/?d=20160328&p=13&s=ni%u011fde";
String expectedURL = "http://www.hurriyet.com.tr/index/?d=20160328&p=13&s=ni%C4%9Fde";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), inputURL);
@@ -301,11 +303,11 @@ class BasicURLNormalizerTest {
}
@Test
- void testHostIDNtoASCII() throws MalformedURLException {
+ void testHostIDNtoASCII() throws MalformedURLException, URISyntaxException {
ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance);
filterParams.put("hostIDNtoASCII", true);
URLFilter urlFilter = createFilter(filterParams);
- URL testSourceUrl = new URL("http://www.example.com/");
+ URL testSourceUrl = new URI("http://www.example.com/").toURL();
String inputURL = "http://señal6.com.ar/";
String expectedURL = "http://xn--seal6-pta.com.ar/";
String normalizedUrl = urlFilter.filter(testSourceUrl, new Metadata(), inputURL);
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/FastURLFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/FastURLFilterTest.java
index 6ae14691..09bd8c31 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/FastURLFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/FastURLFilterTest.java
@@ -19,6 +19,8 @@ package org.apache.stormcrawler.filtering;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
@@ -39,28 +41,28 @@ class FastURLFilterTest {
}
@Test
- void testImagesFilter() throws MalformedURLException {
- URL url = new URL("http://www.somedomain.com/image.jpg");
+ void testImagesFilter() throws MalformedURLException, URISyntaxException {
+ URL url = new URI("http://www.somedomain.com/image.jpg").toURL();
Metadata metadata = new Metadata();
String filterResult = createFilter().filter(url, metadata, url.toExternalForm());
Assertions.assertNull(filterResult);
}
@Test
- void testDomainNotAllowed() throws MalformedURLException {
- URL url = new URL("http://stormcrawler.net/");
+ void testDomainNotAllowed() throws MalformedURLException, URISyntaxException {
+ URL url = new URI("http://stormcrawler.net/").toURL();
Metadata metadata = new Metadata();
String filterResult = createFilter().filter(url, metadata, url.toExternalForm());
Assertions.assertNull(filterResult);
// allowed
- url = new URL("http://stormcrawler.net/bla/");
+ url = new URI("http://stormcrawler.net/bla/").toURL();
filterResult = createFilter().filter(url, metadata, url.toExternalForm());
Assertions.assertEquals(url.toString(), filterResult);
}
@Test
- void testMD() throws MalformedURLException {
- URL url = new URL("http://somedomain.net/");
+ void testMD() throws MalformedURLException, URISyntaxException {
+ URL url = new URI("http://somedomain.net/").toURL();
Metadata metadata = new Metadata();
metadata.addValue("key", "value");
String filterResult = createFilter().filter(url, metadata, url.toExternalForm());
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
index 5b183919..59b81c64 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
@@ -19,6 +19,8 @@ package org.apache.stormcrawler.filtering;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
@@ -44,9 +46,9 @@ class HostURLFilterTest {
}
@Test
- void testAllAllowed() throws MalformedURLException {
+ void testAllAllowed() throws MalformedURLException, URISyntaxException {
HostURLFilter allAllowed = createFilter(false, false);
- URL sourceURL = new URL("http://www.sourcedomain.com/index.html");
+ URL sourceURL = new URI("http://www.sourcedomain.com/index.html").toURL();
Metadata metadata = new Metadata();
String filterResult =
allAllowed.filter(sourceURL, metadata, "http://www.sourcedomain.com/index.html");
@@ -60,9 +62,9 @@ class HostURLFilterTest {
}
@Test
- void testAllForbidden() throws MalformedURLException {
+ void testAllForbidden() throws MalformedURLException, URISyntaxException {
HostURLFilter allAllowed = createFilter(true, true);
- URL sourceURL = new URL("http://www.sourcedomain.com/index.html");
+ URL sourceURL = new URI("http://www.sourcedomain.com/index.html").toURL();
Metadata metadata = new Metadata();
String filterResult =
allAllowed.filter(sourceURL, metadata, "http://www.sourcedomain.com/index.html");
@@ -76,9 +78,9 @@ class HostURLFilterTest {
}
@Test
- void testWithinHostOnly() throws MalformedURLException {
+ void testWithinHostOnly() throws MalformedURLException, URISyntaxException {
HostURLFilter allAllowed = createFilter(true, false);
- URL sourceURL = new URL("http://www.sourcedomain.com/index.html");
+ URL sourceURL = new URI("http://www.sourcedomain.com/index.html").toURL();
Metadata metadata = new Metadata();
String filterResult =
allAllowed.filter(sourceURL, metadata, "http://www.sourcedomain.com/index.html");
@@ -92,9 +94,9 @@ class HostURLFilterTest {
}
@Test
- void testWithinDomain() throws MalformedURLException {
+ void testWithinDomain() throws MalformedURLException, URISyntaxException {
HostURLFilter allAllowed = createFilter(false, true);
- URL sourceURL = new URL("http://www.sourcedomain.com/index.html");
+ URL sourceURL = new URI("http://www.sourcedomain.com/index.html").toURL();
Metadata metadata = new Metadata();
String filterResult =
allAllowed.filter(sourceURL, metadata, "http://www.sourcedomain.com/index.html");
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/MaxDepthFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/MaxDepthFilterTest.java
index a76a5c74..24fa6563 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/MaxDepthFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/MaxDepthFilterTest.java
@@ -19,6 +19,8 @@ package org.apache.stormcrawler.filtering;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
@@ -40,18 +42,18 @@ class MaxDepthFilterTest {
}
@Test
- void testDepthZero() throws MalformedURLException {
+ void testDepthZero() throws MalformedURLException, URISyntaxException {
URLFilter filter = createFilter("maxDepth", 0);
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
String filterResult = filter.filter(url, metadata, url.toExternalForm());
Assertions.assertNull(filterResult);
}
@Test
- void testDepth() throws MalformedURLException {
+ void testDepth() throws MalformedURLException, URISyntaxException {
URLFilter filter = createFilter("maxDepth", 2);
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
metadata.setValue(MetadataTransfer.depthKeyName, "2");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -59,9 +61,9 @@ class MaxDepthFilterTest {
}
@Test
- void testCustomDepthZero() throws MalformedURLException {
+ void testCustomDepthZero() throws MalformedURLException, URISyntaxException {
URLFilter filter = createFilter("maxDepth", 3);
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
metadata.setValue(MetadataTransfer.maxDepthKeyName, "0");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -69,9 +71,9 @@ class MaxDepthFilterTest {
}
@Test
- void testCustomDepth() throws MalformedURLException {
+ void testCustomDepth() throws MalformedURLException, URISyntaxException {
URLFilter filter = createFilter("maxDepth", 1);
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
metadata.setValue(MetadataTransfer.maxDepthKeyName, "2");
metadata.setValue(MetadataTransfer.depthKeyName, "1");
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterFromJsonTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterFromJsonTest.java
index 85a624b9..14ae326b 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterFromJsonTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterFromJsonTest.java
@@ -17,6 +17,8 @@
package org.apache.stormcrawler.filtering;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.Map;
import org.apache.stormcrawler.Metadata;
@@ -31,18 +33,18 @@ class MetadataFilterFromJsonTest {
// old filter mechanism (backward compatible)
@Test
- void testFilterNoMD() throws MalformedURLException {
+ void testFilterNoMD() throws MalformedURLException, URISyntaxException {
URLFilters filter = createURLFilters("test.metadata.1.urlfilters.json");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
String filterResult = filter.filter(url, metadata, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
}
@Test
- void testFilterHit() throws MalformedURLException {
+ void testFilterHit() throws MalformedURLException, URISyntaxException {
URLFilters filter = createURLFilters("test.metadata.1.urlfilters.json");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -50,9 +52,9 @@ class MetadataFilterFromJsonTest {
}
@Test
- void testFilterNoHit() throws MalformedURLException {
+ void testFilterNoHit() throws MalformedURLException, URISyntaxException {
URLFilters filter = createURLFilters("test.metadata.1.urlfilters.json");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
metadata.addValue("key", "val2");
metadata.addValue("key", "val3");
@@ -62,36 +64,36 @@ class MetadataFilterFromJsonTest {
// new filter mechanism
@Test
- void testNewFilterWithEmptyFilterAndNullMetadata() throws MalformedURLException {
+ void testNewFilterWithEmptyFilterAndNullMetadata() throws MalformedURLException, URISyntaxException {
URLFilters filter = createURLFilters("test.metadata.2.urlfilters.json");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
String filterResult = filter.filter(url, null, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
}
@Test
- void testNewFilterWithEmptyFilterAndEmptyMetadata() throws MalformedURLException {
+ void testNewFilterWithEmptyFilterAndEmptyMetadata() throws MalformedURLException, URISyntaxException {
URLFilters filter = createURLFilters("test.metadata.2.urlfilters.json");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
String filterResult = filter.filter(url, metadata, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
}
@Test
- void testNewFilterWithEmptyMetadata() throws MalformedURLException {
+ void testNewFilterWithEmptyMetadata() throws MalformedURLException, URISyntaxException {
URLFilters filter = createURLFilters("test.metadata.2.urlfilters.json");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
String filterResult = filter.filter(url, metadata, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
}
@Test
- void testNewFilterWithOnlyOneMatchingANDFilter() throws MalformedURLException {
+ void testNewFilterWithOnlyOneMatchingANDFilter() throws MalformedURLException, URISyntaxException {
// Filter if key=>val AND key2=>val2 match
URLFilters filter = createURLFilters("test.metadata.2.urlfilters.json");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -99,10 +101,10 @@ class MetadataFilterFromJsonTest {
}
@Test
- void testNewFilterWithAllMatchingANDFilter() throws MalformedURLException {
+ void testNewFilterWithAllMatchingANDFilter() throws MalformedURLException, URISyntaxException {
// Filter if key=>val AND key2=>val2 match
URLFilters filter = createURLFilters("test.metadata.2.urlfilters.json");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
metadata.addValue("key2", "val2");
@@ -111,10 +113,10 @@ class MetadataFilterFromJsonTest {
}
@Test
- void testNewFilterWithComplexFilter() throws MalformedURLException {
+ void testNewFilterWithComplexFilter() throws MalformedURLException, URISyntaxException {
// Filter if key=>val AND (key2=>val2 OR key3=>val3) match
URLFilters filter = createURLFilters("test.metadata.3.urlfilters.json");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterTest.java
index 6a39568b..1a84e9ef 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/MetadataFilterTest.java
@@ -19,6 +19,8 @@ package org.apache.stormcrawler.filtering;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
@@ -40,18 +42,18 @@ class MetadataFilterTest {
// old filter mechanism (backward compatible)
@Test
- void testFilterNoMD() throws MalformedURLException {
+ void testFilterNoMD() throws MalformedURLException, URISyntaxException {
URLFilter filter = createFilter("key", "val");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
String filterResult = filter.filter(url, metadata, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
}
@Test
- void testFilterHit() throws MalformedURLException {
+ void testFilterHit() throws MalformedURLException, URISyntaxException {
URLFilter filter = createFilter("key", "val");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -59,9 +61,9 @@ class MetadataFilterTest {
}
@Test
- void testFilterNoHit() throws MalformedURLException {
+ void testFilterNoHit() throws MalformedURLException, URISyntaxException {
URLFilter filter = createFilter("key", "val");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
metadata.addValue("key", "val2");
metadata.addValue("key", "val3");
@@ -71,26 +73,26 @@ class MetadataFilterTest {
// new filter mechanism
@Test
- void testNewFilterWithEmptyFilterAndNullMetadata() throws MalformedURLException {
+ void testNewFilterWithEmptyFilterAndNullMetadata() throws MalformedURLException, URISyntaxException {
MetadataFilter filter = new MetadataFilter();
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
String filterResult = filter.filter(url, null, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
}
@Test
- void testNewFilterWithEmptyFilterAndEmptyMetadata() throws MalformedURLException {
+ void testNewFilterWithEmptyFilterAndEmptyMetadata() throws MalformedURLException, URISyntaxException {
MetadataFilter filter = new MetadataFilter();
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
String filterResult = filter.filter(url, metadata, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
}
@Test
- void testNewFilterWithEmptyFilter() throws MalformedURLException {
+ void testNewFilterWithEmptyFilter() throws MalformedURLException, URISyntaxException {
MetadataFilter filter = new MetadataFilter();
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -98,21 +100,21 @@ class MetadataFilterTest {
}
@Test
- void testNewFilterWithEmptyMetadata() throws MalformedURLException {
+ void testNewFilterWithEmptyMetadata() throws MalformedURLException, URISyntaxException {
MetadataFilter filter = new MetadataFilter();
filter.addFilter("key", "val");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
String filterResult = filter.filter(url, metadata, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
}
@Test
- void testNewFilterWithSingleMatchingORFilter() throws MalformedURLException {
+ void testNewFilterWithSingleMatchingORFilter() throws MalformedURLException, URISyntaxException {
// Filter if key=>val match (OR operation)
MetadataFilter filter = new MetadataFilter();
filter.addFilter("key", "val");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -120,12 +122,12 @@ class MetadataFilterTest {
}
@Test
- void testNewFilterWithSingleMatchingANDFilter() throws MalformedURLException {
+ void testNewFilterWithSingleMatchingANDFilter() throws MalformedURLException, URISyntaxException {
// Filter if key=>val match (AND operation)
MetadataFilter filter = new MetadataFilter();
filter.addFilter("key", "val");
filter.setOperation(MetadataFilter.FilterOperation.AND);
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -133,12 +135,12 @@ class MetadataFilterTest {
}
@Test
- void testNewFilterWithOnlyOneMatchingORFilter() throws MalformedURLException {
+ void testNewFilterWithOnlyOneMatchingORFilter() throws MalformedURLException, URISyntaxException {
// Filter if key=>val OR key2=>val2 match
MetadataFilter filter = new MetadataFilter();
filter.addFilter("key", "val");
filter.addFilter("key2", "val2");
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -146,13 +148,13 @@ class MetadataFilterTest {
}
@Test
- void testNewFilterWithOnlyOneMatchingANDFilter() throws MalformedURLException {
+ void testNewFilterWithOnlyOneMatchingANDFilter() throws MalformedURLException, URISyntaxException {
// Filter if key=>val AND key2=>val2 match
MetadataFilter filter = new MetadataFilter();
filter.addFilter("key", "val");
filter.addFilter("key2", "val2");
filter.setOperation(MetadataFilter.FilterOperation.AND);
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
String filterResult = filter.filter(url, metadata, url.toExternalForm());
@@ -160,13 +162,13 @@ class MetadataFilterTest {
}
@Test
- void testNewFilterWithAllMatchingANDFilter() throws MalformedURLException {
+ void testNewFilterWithAllMatchingANDFilter() throws MalformedURLException, URISyntaxException {
// Filter if key=>val AND key2=>val2 match
MetadataFilter filter = new MetadataFilter();
filter.addFilter("key", "val");
filter.addFilter("key2", "val2");
filter.setOperation(MetadataFilter.FilterOperation.AND);
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
metadata.addValue("key2", "val2");
@@ -175,7 +177,7 @@ class MetadataFilterTest {
}
@Test
- void testNewFilterWithComplexFilter() throws MalformedURLException {
+ void testNewFilterWithComplexFilter() throws MalformedURLException, URISyntaxException {
// Filter if key=>val AND (key2=>val2 OR key3=>val3) match
MetadataFilter filter = new MetadataFilter();
filter.addFilter("key", "val");
@@ -184,7 +186,7 @@ class MetadataFilterTest {
filter2.addFilter("key2", "val2");
filter2.addFilter("key3", "val3");
filter.addFilter(filter2);
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
@@ -221,7 +223,7 @@ class MetadataFilterTest {
}
@Test
- void testNewFilterWithOtherComplexFilter() throws MalformedURLException {
+ void testNewFilterWithOtherComplexFilter() throws MalformedURLException, URISyntaxException {
// Filter if key=>val OR (key2=>val2 AND key3=>val3) match
MetadataFilter filter = new MetadataFilter();
filter.addFilter("key", "val");
@@ -230,7 +232,7 @@ class MetadataFilterTest {
filter2.addFilter("key3", "val3");
filter2.setOperation(MetadataFilter.FilterOperation.AND);
filter.addFilter(filter2);
- URL url = new URL("http://www.sourcedomain.com/");
+ URL url = new URI("http://www.sourcedomain.com/").toURL();
Metadata metadata = new Metadata();
metadata.addValue("key", "val");
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/RegexFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/RegexFilterTest.java
index 60482433..183495ae 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/RegexFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/RegexFilterTest.java
@@ -19,6 +19,8 @@ package org.apache.stormcrawler.filtering;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
@@ -43,35 +45,35 @@ class RegexFilterTest {
}
@Test
- void testProtocolFilter() throws MalformedURLException {
+ void testProtocolFilter() throws MalformedURLException, URISyntaxException {
URLFilter allAllowed = createFilter();
- URL url = new URL("ftp://www.someFTP.com/#0");
+ URL url = new URI("ftp://www.someFTP.com/#0").toURL();
Metadata metadata = new Metadata();
String filterResult = allAllowed.filter(url, metadata, url.toExternalForm());
Assertions.assertNull(filterResult);
}
@Test
- void testImagesFilter() throws MalformedURLException {
+ void testImagesFilter() throws MalformedURLException, URISyntaxException {
URLFilter allAllowed = createFilter();
- URL url = new URL("http://www.someFTP.com/bla.gif");
+ URL url = new URI("http://www.someFTP.com/bla.gif").toURL();
Metadata metadata = new Metadata();
String filterResult = allAllowed.filter(url, metadata, url.toExternalForm());
Assertions.assertNull(filterResult);
- url = new URL("http://www.someFTP.com/bla.GIF");
+ url = new URI("http://www.someFTP.com/bla.GIF").toURL();
filterResult = allAllowed.filter(url, metadata, url.toExternalForm());
Assertions.assertNull(filterResult);
- url = new URL("http://www.someFTP.com/bla.GIF&somearg=0");
+ url = new URI("http://www.someFTP.com/bla.GIF&somearg=0").toURL();
filterResult = allAllowed.filter(url, metadata, url.toExternalForm());
Assertions.assertNull(filterResult);
- url = new URL("http://www.someFTP.com/bla.GIF?somearg=0");
+ url = new URI("http://www.someFTP.com/bla.GIF?somearg=0").toURL();
filterResult = allAllowed.filter(url, metadata, url.toExternalForm());
Assertions.assertNull(filterResult);
// not this one : the gif is within the path
- url = new URL("http://www.someFTP.com/bla.GIF.orNot");
+ url = new URI("http://www.someFTP.com/bla.GIF.orNot").toURL();
filterResult = allAllowed.filter(url, metadata, url.toExternalForm());
Assertions.assertEquals(url.toExternalForm(), filterResult);
- url = new URL("http://www.someFTP.com/bla.mp4");
+ url = new URI("http://www.someFTP.com/bla.mp4").toURL();
filterResult = allAllowed.filter(url, metadata, url.toExternalForm());
Assertions.assertNull(filterResult);
}
diff --git a/core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java b/core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java
index 1b9cca8d..a434a825 100644
--- a/core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/util/CookieConverterTest.java
@@ -17,6 +17,8 @@
package org.apache.stormcrawler.util;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.List;
import org.apache.http.cookie.Cookie;
@@ -307,8 +309,8 @@ class CookieConverterTest {
private URL getUrl(String urlString) {
try {
- return new URL(urlString);
- } catch (MalformedURLException e) {
+ return new URI(urlString).toURL();
+ } catch (MalformedURLException | URISyntaxException e) {
return null;
}
}
diff --git a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java b/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java
index 219f4b0a..35ef66b1 100644
--- a/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java
+++ b/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/RemoteDriverProtocol.java
@@ -16,6 +16,7 @@
*/
package org.apache.stormcrawler.protocol.selenium;
+import java.net.URI;
import java.net.URL;
import java.time.Duration;
import java.util.ArrayList;
@@ -98,7 +99,7 @@ public class RemoteDriverProtocol extends SeleniumProtocol {
for (String cdaddress : addresses) {
try {
RemoteWebDriver driver =
- new RemoteWebDriver(new URL(cdaddress), capabilities, tracing);
+ new RemoteWebDriver(new URI(cdaddress).toURL(), capabilities, tracing);
// setting timouts
// see https://www.browserstack.com/guide/understanding-selenium-timeouts
Timeouts touts = driver.manage().timeouts();
diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
index 09dea259..7e229848 100644
--- a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
+++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
@@ -21,6 +21,8 @@ import static org.apache.stormcrawler.Constants.StatusStreamName;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
@@ -204,9 +206,9 @@ public class ParserBolt extends BaseRichBolt {
// as well as the filename
try {
- URL _url = new URL(url);
+ URL _url = new URI(url).toURL();
md.set(TikaCoreProperties.RESOURCE_NAME_KEY, _url.getFile());
- } catch (MalformedURLException e1) {
+ } catch (MalformedURLException | URISyntaxException e1) {
throw new IllegalStateException("Malformed URL", e1);
}
diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCSpout.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCSpout.java
index fb2e5a1c..df127f58 100644
--- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCSpout.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCSpout.java
@@ -17,6 +17,8 @@
package org.apache.stormcrawler.warc;
import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
@@ -145,9 +147,9 @@ public class WARCSpout extends FileSpout {
}
}
- private ReadableByteChannel openChannel(String path) throws IOException {
+ private ReadableByteChannel openChannel(String path) throws IOException, URISyntaxException {
if (path.matches("^https?://.*")) {
- URL warcUrl = new URL(path);
+ URL warcUrl = new URI(path).toURL();
return Channels.newChannel(warcUrl.openStream());
}
org.apache.hadoop.fs.Path hdfsPath = new org.apache.hadoop.fs.Path(path);