You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by jn...@apache.org on 2026/07/27 15:55:21 UTC
(stormcrawler) branch main updated: #2026 - http.agent.name is dropped from the robots.txt match set when http.robots.agents is set (#2027)
This is an automated email from the ASF dual-hosted git repository.
jnioche pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
The following commit(s) were added to refs/heads/main by this push:
new d193e6cd #2026 - http.agent.name is dropped from the robots.txt match set when http.robots.agents is set (#2027)
d193e6cd is described below
commit d193e6cdf0fd9562de304895b0391a5630ce9bb8
Author: Richard Zowalla <13...@users.noreply.github.com>
AuthorDate: Mon Jul 27 17:55:18 2026 +0200
#2026 - http.agent.name is dropped from the robots.txt match set when http.robots.agents is set (#2027)
* #2026 - http.agent.name is dropped from the robots.txt match set when http.robots.agents is set
* #2026 - Always match http.agent.name against robots.txt, even when http.robots.agents is set
---
.../stormcrawler/protocol/RobotRulesParser.java | 2 +
.../protocol/RobotRulesParserAgentNamesTest.java | 94 ++++++++++++++++++++++
2 files changed, 96 insertions(+)
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java
index 20c4b22e..d6c36673 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java
@@ -141,6 +141,8 @@ public abstract class RobotRulesParser {
agentName);
this.agentNames.add(agentName.toLowerCase(Locale.ROOT));
} else {
+ // our agent-string is always the first one we match against robots.txt
+ agentNames.add(agentName);
int index = 0;
if ((agents.get(0)).equalsIgnoreCase(agentName)) {
index++;
diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/RobotRulesParserAgentNamesTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/RobotRulesParserAgentNamesTest.java
new file mode 100644
index 00000000..9e0b302e
--- /dev/null
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/RobotRulesParserAgentNamesTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.protocol;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import crawlercommons.robots.BaseRobotRules;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.storm.Config;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Checks that the agent name advertised via {@code http.agent.name} is always part of the set of
+ * names used to select the matching groups in a robots.txt, regardless of the content of {@code
+ * http.robots.agents}.
+ */
+class RobotRulesParserAgentNamesTest {
+
+ private static final String ROBOTS_TXT =
+ "User-agent: mybot\n"
+ + "Disallow: /restricted/\n"
+ + "\n"
+ + "User-agent: *\n"
+ + "Disallow: /\n";
+
+ private static RobotRulesParser parserWith(String agentName, String robotsAgents) {
+ Config conf = new Config();
+ conf.put("http.agent.name", agentName);
+ if (robotsAgents != null) {
+ conf.put("http.robots.agents", robotsAgents);
+ }
+ RobotRulesParser parser = new HttpRobotRulesParser();
+ parser.setConf(conf);
+ return parser;
+ }
+
+ @Test
+ void agentNameUsedWhenNoRobotsAgentsConfigured() {
+ RobotRulesParser parser = parserWith("mybot", null);
+ Assertions.assertEquals(List.of("mybot"), List.copyOf(parser.agentNames));
+ }
+
+ @Test
+ void agentNameKeptWhenListedFirst() {
+ RobotRulesParser parser = parserWith("mybot", "mybot,mybot-uk");
+ Assertions.assertEquals(
+ Arrays.asList("mybot", "mybot-uk"),
+ List.copyOf(parser.agentNames),
+ "http.agent.name must be the first name matched against robots.txt");
+ }
+
+ @Test
+ void agentNameAddedWhenNotListedFirst() {
+ RobotRulesParser parser = parserWith("mybot", "mybot-uk,other-bot");
+ Assertions.assertTrue(
+ parser.agentNames.contains("mybot"),
+ "http.agent.name must be matched against robots.txt even when not listed first, but got "
+ + parser.agentNames);
+ }
+
+ @Test
+ void groupAddressedToAdvertisedAgentIsSelected() {
+ RobotRulesParser parser = parserWith("mybot", "mybot,mybot-uk");
+ BaseRobotRules rules =
+ parser.parseRules(
+ "http://example.com/robots.txt",
+ ROBOTS_TXT.getBytes(UTF_8),
+ "text/plain",
+ parser.agentNames);
+ // the "User-agent: mybot" group only disallows /restricted/, the wildcard group
+ // disallows everything - if the advertised name is dropped we fall back to "*"
+ Assertions.assertTrue(
+ rules.isAllowed("http://example.com/index.html"),
+ "fell back to the wildcard group instead of the group for 'mybot'");
+ Assertions.assertFalse(rules.isAllowed("http://example.com/restricted/index.html"));
+ }
+}