(stormcrawler) 01/01: Honour ignoreOutsideDomain when set on its own in HostURLFilter

[rz...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

rzo1 pushed a commit to branch fix/hosturlfilter-domain-config
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git

commit 47f31637416d9a70f2f68e9b540f83e0b10546f9
Author: Richard Zowalla <13...@users.noreply.github.com>
AuthorDate: Thu Aug 27 14:20:10 2026 +0200

    Honour ignoreOutsideDomain when set on its own in HostURLFilter
    
    HostURLFilter.configure() read the ignoreOutsideDomain parameter but tested
    the ignoreOutsideHost node for null. A configuration naming only
    ignoreOutsideDomain was therefore discarded and the filter let every URL
    through, while one naming only ignoreOutsideHost failed with a
    NullPointerException while the topology was starting. Now each parameter is
    tested independently. Topologies which set only ignoreOutsideDomain start
    restricting the crawl to the source domain, as configured.
---
 .../stormcrawler/filtering/host/HostURLFilter.java |  2 +-
 .../stormcrawler/filtering/HostURLFilterTest.java  | 37 +++++++++++++++++++++-
 2 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java
index 645b7b6d..d14541fc 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java
@@ -62,7 +62,7 @@ public class HostURLFilter extends URLFilter {
         // always the same
         if (!ignoreOutsideHost) {
             JsonNode filterByDomainNode = filterParams.get("ignoreOutsideDomain");
-            if (filterByHostNode == null) {
+            if (filterByDomainNode == null) {
                 ignoreOutsideDomain = false;
             } else {
                 ignoreOutsideDomain = filterByDomainNode.asBoolean(false);
diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
index f70ca90b..c4457903 100644
--- a/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
@@ -36,10 +36,14 @@ import org.junit.jupiter.api.Test;
 class HostURLFilterTest {
 
     private HostURLFilter createFilter(boolean ignoreOutsideHost, boolean ignoreOutsideDomain) {
-        HostURLFilter filter = new HostURLFilter();
         ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance);
         filterParams.put("ignoreOutsideHost", Boolean.valueOf(ignoreOutsideHost));
         filterParams.put("ignoreOutsideDomain", Boolean.valueOf(ignoreOutsideDomain));
+        return createFilter(filterParams);
+    }
+
+    private HostURLFilter createFilter(ObjectNode filterParams) {
+        HostURLFilter filter = new HostURLFilter();
         Map<String, Object> conf = new HashMap<>();
         filter.configure(conf, filterParams);
         return filter;
@@ -108,4 +112,35 @@ class HostURLFilterTest {
                 allAllowed.filter(sourceURL, metadata, "http://sub.sourcedomain.com/index.html");
         Assertions.assertEquals("http://sub.sourcedomain.com/index.html", filterResult);
     }
+
+    /** The two modes are independent, so ignoreOutsideDomain must be honoured on its own. */
+    @Test
+    void testWithinDomainWithoutHostParameter() throws MalformedURLException {
+        ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance);
+        filterParams.put("ignoreOutsideDomain", Boolean.TRUE);
+        HostURLFilter withinDomain = createFilter(filterParams);
+        URL sourceURL = URLUtil.toURL("http://www.sourcedomain.com/index.html");
+        Metadata metadata = new Metadata();
+        String filterResult =
+                withinDomain.filter(sourceURL, metadata, "http://sub.sourcedomain.com/index.html");
+        Assertions.assertEquals("http://sub.sourcedomain.com/index.html", filterResult);
+        filterResult =
+                withinDomain.filter(sourceURL, metadata, "http://www.anotherDomain.com/index.html");
+        Assertions.assertNull(filterResult);
+    }
+
+    /**
+     * A configuration which sets ignoreOutsideHost only must not fail on the missing domain key.
+     */
+    @Test
+    void testHostParameterWithoutDomainParameter() throws MalformedURLException {
+        ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance);
+        filterParams.put("ignoreOutsideHost", Boolean.FALSE);
+        HostURLFilter allAllowed = createFilter(filterParams);
+        URL sourceURL = URLUtil.toURL("http://www.sourcedomain.com/index.html");
+        Metadata metadata = new Metadata();
+        String filterResult =
+                allAllowed.filter(sourceURL, metadata, "http://www.anotherDomain.com/index.html");
+        Assertions.assertEquals("http://www.anotherDomain.com/index.html", filterResult);
+    }
 }