(stormcrawler) branch fix/hosturlfilter-domain-config created (now 47f31637)

[rz...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

rzo1 pushed a change to branch fix/hosturlfilter-domain-config
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git


      at 47f31637 Honour ignoreOutsideDomain when set on its own in HostURLFilter

This branch includes the following new commits:

     new 47f31637 Honour ignoreOutsideDomain when set on its own in HostURLFilter

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

(stormcrawler) 01/01: Honour ignoreOutsideDomain when set on its own in HostURLFilter

[rz...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

rzo1 pushed a commit to branch fix/hosturlfilter-domain-config
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git

commit 47f31637416d9a70f2f68e9b540f83e0b10546f9
Author: Richard Zowalla <13...@users.noreply.github.com>
AuthorDate: Thu Aug 27 14:20:10 2026 +0200

    Honour ignoreOutsideDomain when set on its own in HostURLFilter
    
    HostURLFilter.configure() read the ignoreOutsideDomain parameter but tested
    the ignoreOutsideHost node for null. A configuration naming only
    ignoreOutsideDomain was therefore discarded and the filter let every URL
    through, while one naming only ignoreOutsideHost failed with a
    NullPointerException while the topology was starting. Now each parameter is
    tested independently. Topologies which set only ignoreOutsideDomain start
    restricting the crawl to the source domain, as configured.
---
 .../stormcrawler/filtering/host/HostURLFilter.java |  2 +-
 .../stormcrawler/filtering/HostURLFilterTest.java  | 37 +++++++++++++++++++++-
 2 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java
index 645b7b6d..d14541fc 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java
@@ -62,7 +62,7 @@ public class HostURLFilter extends URLFilter {
         // always the same
         if (!ignoreOutsideHost) {
             JsonNode filterByDomainNode = filterParams.get("ignoreOutsideDomain");
-            if (filterByHostNode == null) {
+            if (filterByDomainNode == null) {
                 ignoreOutsideDomain = false;
             } else {
                 ignoreOutsideDomain = filterByDomainNode.asBoolean(false);
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
index f70ca90b..c4457903 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
@@ -36,10 +36,14 @@ import org.junit.jupiter.api.Test;
 class HostURLFilterTest {
 
     private HostURLFilter createFilter(boolean ignoreOutsideHost, boolean ignoreOutsideDomain) {
-        HostURLFilter filter = new HostURLFilter();
         ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance);
         filterParams.put("ignoreOutsideHost", Boolean.valueOf(ignoreOutsideHost));
         filterParams.put("ignoreOutsideDomain", Boolean.valueOf(ignoreOutsideDomain));
+        return createFilter(filterParams);
+    }
+
+    private HostURLFilter createFilter(ObjectNode filterParams) {
+        HostURLFilter filter = new HostURLFilter();
         Map<String, Object> conf = new HashMap<>();
         filter.configure(conf, filterParams);
         return filter;
@@ -108,4 +112,35 @@ class HostURLFilterTest {
                 allAllowed.filter(sourceURL, metadata, "http://sub.sourcedomain.com/index.html");
         Assertions.assertEquals("http://sub.sourcedomain.com/index.html", filterResult);
     }
+
+    /** The two modes are independent, so ignoreOutsideDomain must be honoured on its own. */
+    @Test
+    void testWithinDomainWithoutHostParameter() throws MalformedURLException {
+        ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance);
+        filterParams.put("ignoreOutsideDomain", Boolean.TRUE);
+        HostURLFilter withinDomain = createFilter(filterParams);
+        URL sourceURL = URLUtil.toURL("http://www.sourcedomain.com/index.html");
+        Metadata metadata = new Metadata();
+        String filterResult =
+                withinDomain.filter(sourceURL, metadata, "http://sub.sourcedomain.com/index.html");
+        Assertions.assertEquals("http://sub.sourcedomain.com/index.html", filterResult);
+        filterResult =
+                withinDomain.filter(sourceURL, metadata, "http://www.anotherDomain.com/index.html");
+        Assertions.assertNull(filterResult);
+    }
+
+    /**
+     * A configuration which sets ignoreOutsideHost only must not fail on the missing domain key.
+     */
+    @Test
+    void testHostParameterWithoutDomainParameter() throws MalformedURLException {
+        ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance);
+        filterParams.put("ignoreOutsideHost", Boolean.FALSE);
+        HostURLFilter allAllowed = createFilter(filterParams);
+        URL sourceURL = URLUtil.toURL("http://www.sourcedomain.com/index.html");
+        Metadata metadata = new Metadata();
+        String filterResult =
+                allAllowed.filter(sourceURL, metadata, "http://www.anotherDomain.com/index.html");
+        Assertions.assertEquals("http://www.anotherDomain.com/index.html", filterResult);
+    }
 }