You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by rz...@apache.org on 2026/09/08 19:07:41 UTC
(stormcrawler) branch main updated: IPFilterRules: add anylocal and multicast keywords, fail on unparseable rules (#2119)
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
The following commit(s) were added to refs/heads/main by this push:
new c3fa48bb IPFilterRules: add anylocal and multicast keywords, fail on unparseable rules (#2119)
c3fa48bb is described below
commit c3fa48bbbea14ad843df2c10937ab8e41a161383
Author: Abhinav <al...@gmail.com>
AuthorDate: Wed Sep 9 00:37:36 2026 +0530
IPFilterRules: add anylocal and multicast keywords, fail on unparseable rules (#2119)
* IPFilterRules: add anylocal and multicast keywords, fail on unparseable rules (#2086)
A rule that is neither a keyword, a CIDR block nor a single address was
logged and dropped, silently widening what the crawler will connect to.
Configuration now fails with an IllegalArgumentException instead, so a
typo in an exclude list stops the topology rather than removing a rule
the operator believed was in force.
Adds the anylocal and multicast keywords, documents linklocal (already
present but missing from the javadoc), and updates the example in
crawler-default.yaml to also cover CGNAT and IPv6 unique-local space.
* Example covers all keywords and the full ULA range (#2086)
The commented example in crawler-default.yaml now exercises anylocal
and multicast and uses fc00::/7 (the whole unique-local range), not
just the locally-assigned fd00::/8 half. The documented-example test
follows.
* Normalize imports and formatting to satisfy CI
The import groups are collapsed into the single ASCII-sorted group the
project checkstyle config demands (STATIC###THIRD_PARTY_PACKAGE), the
constructors in URLFilters are grouped as checkstyle requires, and the
touched files are reformatted with the google-java-format version the
git-code-format plugin pins (1.35.0, AOSP). No code changes.
---
.../stormcrawler/protocol/IPFilterRules.java | 27 +++++++++++++---
core/src/main/resources/crawler-default.yaml | 6 +++-
.../stormcrawler/protocol/IPFilterRulesTest.java | 36 ++++++++++++++++++++--
3 files changed, 60 insertions(+), 9 deletions(-)
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java b/core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java
index 2fbb5aec..5b35ea2c 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java
@@ -49,10 +49,18 @@ import org.slf4j.LoggerFactory;
* {@link InetAddress#isLoopbackAddress()} is true
* <li><code>sitelocal</code> applies to all IP addresses for which {@link
* InetAddress#isSiteLocalAddress()} is true
+ * <li><code>linklocal</code> applies to all IP addresses for which {@link
+ * InetAddress#isLinkLocalAddress()} is true
+ * <li><code>anylocal</code> applies to all IP addresses for which {@link
+ * InetAddress#isAnyLocalAddress()} is true
+ * <li><code>multicast</code> applies to all IP addresses for which {@link
+ * InetAddress#isMulticastAddress()} is true
* </ul>
*
* <p>Multiple IP ranges can be given either as a comma-separated string, e.g. <code>
- * loopback,sitelocal,fd00::/8</code>, or as a list in the configuration.
+ * loopback,sitelocal,fd00::/8</code>, or as a list in the configuration. A value which is neither a
+ * keyword, a CIDR block nor a single IP address is a configuration error and throws an {@link
+ * IllegalArgumentException}.
*/
public class IPFilterRules {
@@ -119,15 +127,24 @@ public class IPFilterRules {
case "linklocal":
rules.add(InetAddress::isLinkLocalAddress);
break;
+ case "anylocal":
+ rules.add(InetAddress::isAnyLocalAddress);
+ break;
+ case "multicast":
+ rules.add(InetAddress::isMulticastAddress);
+ break;
default:
try {
CIDR cidr = new CIDR(ipRule);
rules.add(cidr::contains);
} catch (IllegalArgumentException e) {
- LOG.error(
- "Failed to parse {} as CIDR, ignoring it while configuring IP rules ({})",
- ipRule,
- ipRuleProperty);
+ throw new IllegalArgumentException(
+ "Failed to parse "
+ + ipRule
+ + " as CIDR while configuring IP rules ("
+ + ipRuleProperty
+ + ")",
+ e);
}
}
}
diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml
index 6945b2c4..2f49b51c 100644
--- a/core/src/main/resources/crawler-default.yaml
+++ b/core/src/main/resources/crawler-default.yaml
@@ -159,10 +159,14 @@ config:
# - "localhost" / "loopback" (matches InetAddress.isLoopbackAddress())
# - "sitelocal" (matches InetAddress.isSiteLocalAddress())
# - "linklocal" (matches InetAddress.isLinkLocalAddress())
+ # - "anylocal" (matches InetAddress.isAnyLocalAddress())
+ # - "multicast" (matches InetAddress.isMulticastAddress())
+ # A value which is neither of these is a configuration error and stops the
+ # topology from starting.
# Only addresses matching an include rule are fetched (empty means all are
# allowed), addresses matching an exclude rule are always blocked.
# http.filter.ipaddress.include:
- # http.filter.ipaddress.exclude: "localhost,sitelocal,linklocal"
+ # http.filter.ipaddress.exclude: "localhost,sitelocal,linklocal,anylocal,multicast,100.64.0.0/10,fc00::/7"
# Allow all if robots.txt cannot be parsed due to code 403 (Forbidden):
http.robots.403.allow: true
diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/IPFilterRulesTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/IPFilterRulesTest.java
index fa42d190..fce59fc6 100644
--- a/core/src/test/java/org/apache/stormcrawler/protocol/IPFilterRulesTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/IPFilterRulesTest.java
@@ -25,6 +25,7 @@ import java.net.InetAddress;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
class IPFilterRulesTest {
@@ -104,9 +105,38 @@ class IPFilterRulesTest {
}
@Test
- void invalidRuleIsIgnored() {
- IPFilterRules r = rules(null, "not-an-ip,localhost");
- assertFalse(r.accept(ip("127.0.0.1")));
+ void invalidRuleFailsConfiguration() {
+ Assertions.assertThrows(
+ IllegalArgumentException.class, () -> rules(null, "not-an-ip,localhost"));
+ }
+
+ @Test
+ void documentedExampleBlocksTheNonRoutableRanges() {
+ // the example shipped in crawler-default.yaml
+ IPFilterRules r =
+ rules(
+ null,
+ "localhost,sitelocal,linklocal,anylocal,multicast,100.64.0.0/10,fc00::/7");
+ assertFalse(r.accept(ip("127.0.0.1")), "loopback");
+ assertFalse(r.accept(ip("10.0.0.1")), "RFC1918");
+ assertFalse(r.accept(ip("192.168.1.10")), "RFC1918");
+ assertFalse(r.accept(ip("172.16.0.1")), "RFC1918");
+ assertFalse(r.accept(ip("169.254.169.254")), "IPv4 link-local");
+ assertFalse(r.accept(ip("100.64.0.1")), "CGNAT");
+ assertFalse(r.accept(ip("fd00::1")), "IPv6 unique local, locally assigned half");
+ assertFalse(r.accept(ip("fc00::1")), "IPv6 unique local, centrally assigned half");
+ assertFalse(r.accept(ip("fe80::1")), "IPv6 link-local");
+ assertFalse(r.accept(ip("0.0.0.0")), "anylocal");
+ assertFalse(r.accept(ip("224.0.0.1")), "multicast");
+ assertTrue(r.accept(ip("8.8.8.8")), "public IPv4");
+ }
+
+ @Test
+ void anylocalAndMulticastKeywords() {
+ IPFilterRules r = rules(null, "anylocal,multicast");
+ assertFalse(r.accept(ip("0.0.0.0")), "wildcard");
+ assertFalse(r.accept(ip("::")), "IPv6 wildcard");
+ assertFalse(r.accept(ip("224.0.0.1")), "multicast");
assertTrue(r.accept(ip("8.8.8.8")));
}