You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by rz...@apache.org on 2026/07/27 11:28:13 UTC
(stormcrawler) branch robots-agent-name-dropped created (now 187b9521)
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a change to branch robots-agent-name-dropped
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
at 187b9521 #2026 - Always match http.agent.name against robots.txt, even when http.robots.agents is set
This branch includes the following new commits:
new de2d7f30 #2026 - http.agent.name is dropped from the robots.txt match set when http.robots.agents is set
new 187b9521 #2026 - Always match http.agent.name against robots.txt, even when http.robots.agents is set
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(stormcrawler) 02/02: #2026 - Always match http.agent.name against robots.txt, even when http.robots.agents is set
Posted by rz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch robots-agent-name-dropped
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
commit 187b9521001ade81237ff99d13f3b442128afd54
Author: Richard Zowalla <rz...@apache.org>
AuthorDate: Mon Jul 27 13:28:01 2026 +0200
#2026 - Always match http.agent.name against robots.txt, even when http.robots.agents is set
---
.../main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java | 2 ++
1 file changed, 2 insertions(+)
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java
index 20c4b22e..d6c36673 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/RobotRulesParser.java
@@ -141,6 +141,8 @@ public abstract class RobotRulesParser {
agentName);
this.agentNames.add(agentName.toLowerCase(Locale.ROOT));
} else {
+ // our agent-string is always the first one we match against robots.txt
+ agentNames.add(agentName);
int index = 0;
if ((agents.get(0)).equalsIgnoreCase(agentName)) {
index++;
(stormcrawler) 01/02: #2026 - http.agent.name is dropped from the robots.txt match set when http.robots.agents is set
Posted by rz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch robots-agent-name-dropped
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
commit de2d7f3043947edc8545bb8f44337cafcae3d823
Author: Richard Zowalla <rz...@apache.org>
AuthorDate: Mon Jul 27 13:25:33 2026 +0200
#2026 - http.agent.name is dropped from the robots.txt match set when http.robots.agents is set
---
.../protocol/RobotRulesParserAgentNamesTest.java | 94 ++++++++++++++++++++++
1 file changed, 94 insertions(+)
diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/RobotRulesParserAgentNamesTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/RobotRulesParserAgentNamesTest.java
new file mode 100644
index 00000000..9e0b302e
--- /dev/null
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/RobotRulesParserAgentNamesTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.protocol;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import crawlercommons.robots.BaseRobotRules;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.storm.Config;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Checks that the agent name advertised via {@code http.agent.name} is always part of the set of
+ * names used to select the matching groups in a robots.txt, regardless of the content of {@code
+ * http.robots.agents}.
+ */
+class RobotRulesParserAgentNamesTest {
+
+ private static final String ROBOTS_TXT =
+ "User-agent: mybot\n"
+ + "Disallow: /restricted/\n"
+ + "\n"
+ + "User-agent: *\n"
+ + "Disallow: /\n";
+
+ private static RobotRulesParser parserWith(String agentName, String robotsAgents) {
+ Config conf = new Config();
+ conf.put("http.agent.name", agentName);
+ if (robotsAgents != null) {
+ conf.put("http.robots.agents", robotsAgents);
+ }
+ RobotRulesParser parser = new HttpRobotRulesParser();
+ parser.setConf(conf);
+ return parser;
+ }
+
+ @Test
+ void agentNameUsedWhenNoRobotsAgentsConfigured() {
+ RobotRulesParser parser = parserWith("mybot", null);
+ Assertions.assertEquals(List.of("mybot"), List.copyOf(parser.agentNames));
+ }
+
+ @Test
+ void agentNameKeptWhenListedFirst() {
+ RobotRulesParser parser = parserWith("mybot", "mybot,mybot-uk");
+ Assertions.assertEquals(
+ Arrays.asList("mybot", "mybot-uk"),
+ List.copyOf(parser.agentNames),
+ "http.agent.name must be the first name matched against robots.txt");
+ }
+
+ @Test
+ void agentNameAddedWhenNotListedFirst() {
+ RobotRulesParser parser = parserWith("mybot", "mybot-uk,other-bot");
+ Assertions.assertTrue(
+ parser.agentNames.contains("mybot"),
+ "http.agent.name must be matched against robots.txt even when not listed first, but got "
+ + parser.agentNames);
+ }
+
+ @Test
+ void groupAddressedToAdvertisedAgentIsSelected() {
+ RobotRulesParser parser = parserWith("mybot", "mybot,mybot-uk");
+ BaseRobotRules rules =
+ parser.parseRules(
+ "http://example.com/robots.txt",
+ ROBOTS_TXT.getBytes(UTF_8),
+ "text/plain",
+ parser.agentNames);
+ // the "User-agent: mybot" group only disallows /restricted/, the wildcard group
+ // disallows everything - if the advertised name is dropped we fall back to "*"
+ Assertions.assertTrue(
+ rules.isAllowed("http://example.com/index.html"),
+ "fell back to the wildcard group instead of the group for 'mybot'");
+ Assertions.assertFalse(rules.isAllowed("http://example.com/restricted/index.html"));
+ }
+}