(stormcrawler) 02/08: feat: report the forced robots crawl-delay from SimpleFetcherBolt too

[dp...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dpol1 pushed a commit to branch feat/1979-robots-delay
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git

commit bbf317b51064fa8f8f2de059a2097c3dd967d219
Author: Davide Polato <dp...@apache.org>
AuthorDate: Wed Jul 8 15:43:16 2026 +0200

    feat: report the forced robots crawl-delay from SimpleFetcherBolt too
    
    Same signal as in FetcherBolt: when maxCrawlDelayForce caps a long
    Crawl-delay, the original robots value lands in robots.crawl.delay so
    the frontier side of #1979 sees both fetchers alike. The shared test
    harness moves up to AbstractFetcherBoltTest rather than being copied.
---
 .../stormcrawler/bolt/SimpleFetcherBolt.java       |  5 ++
 .../stormcrawler/bolt/SimpleFetcherBoltTest.java   | 54 ++++++++++++++++++++++
 2 files changed, 59 insertions(+)

diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
index da40b069..2c47dee1 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
@@ -407,6 +407,11 @@ public class SimpleFetcherBolt extends StatusEmitterBolt {
                         // as some sites specify ridiculous values
                         LOG.debug("Delay from robots capped at {} for {}", robotsDelay, url);
                         delay = maxCrawlDelay;
+                        // report the delay the fetcher is not holding, so a frontier-side
+                        // consumer can enforce it at the source (#867)
+                        metadata.setValue(
+                                Constants.ROBOTS_CRAWL_DELAY_KEY,
+                                Long.toString(robotsDelay / 1000L));
                     } else {
                         LOG.debug(
                                 "Skipped URL from queue with overlong crawl-delay ({}): {}",
diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/SimpleFetcherBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/SimpleFetcherBoltTest.java
index da212022..91565228 100644
--- a/core/src/test/java/org/apache/stormcrawler/bolt/SimpleFetcherBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/SimpleFetcherBoltTest.java
@@ -17,7 +17,20 @@
 
 package org.apache.stormcrawler.bolt;
 
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 
 public class SimpleFetcherBoltTest extends AbstractFetcherBoltTest {
 
@@ -25,4 +38,45 @@ public class SimpleFetcherBoltTest extends AbstractFetcherBoltTest {
     void setUpContext() throws Exception {
         bolt = new SimpleFetcherBolt();
     }
+
+    @Test
+    void forcedLongCrawlDelayIsReportedInMetadata(WireMockRuntimeInfo wmRuntimeInfo)
+            throws ReflectiveOperationException {
+        // robots.txt with a Crawl-delay above fetcher.max.crawl.delay
+        stubFor(
+                get(urlEqualTo("/robots.txt"))
+                        .willReturn(
+                                aResponse()
+                                        .withStatus(200)
+                                        .withBody("User-agent: *\nCrawl-delay: 120\n")));
+        stubFor(get(urlEqualTo("/page")).willReturn(aResponse().withStatus(200).withBody("hello")));
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("http.agent.name", "this_is_only_a_test");
+        config.put("fetcher.max.crawl.delay", 30);
+        config.put("fetcher.max.crawl.delay.force", true);
+
+        Metadata md = fetchAndGetContentMetadata(wmRuntimeInfo, config, "/page");
+        assertEquals("120", md.getFirstValue(Constants.ROBOTS_CRAWL_DELAY_KEY));
+    }
+
+    @Test
+    void shortCrawlDelayIsNotReported(WireMockRuntimeInfo wmRuntimeInfo)
+            throws ReflectiveOperationException {
+        stubFor(
+                get(urlEqualTo("/robots.txt"))
+                        .willReturn(
+                                aResponse()
+                                        .withStatus(200)
+                                        .withBody("User-agent: *\nCrawl-delay: 5\n")));
+        stubFor(get(urlEqualTo("/page")).willReturn(aResponse().withStatus(200).withBody("hello")));
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("http.agent.name", "this_is_only_a_test");
+        config.put("fetcher.max.crawl.delay", 30);
+        config.put("fetcher.max.crawl.delay.force", true);
+
+        Metadata md = fetchAndGetContentMetadata(wmRuntimeInfo, config, "/page");
+        assertNull(md.getFirstValue(Constants.ROBOTS_CRAWL_DELAY_KEY));
+    }
 }