You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by rz...@apache.org on 2026/09/19 16:35:56 UTC

(stormcrawler) branch main updated: #2090 Fetcher bolts: drop a set-header value coming from a response (#2168)

This is an automated email from the ASF dual-hosted git repository.

rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git


The following commit(s) were added to refs/heads/main by this push:
     new 379d81c4 #2090 Fetcher bolts: drop a set-header value coming from a response (#2168)
379d81c4 is described below

commit 379d81c4f8f50f720aa2856cdab212d07b0234b1
Author: Richard Zowalla <13...@users.noreply.github.com>
AuthorDate: Sat Sep 19 18:35:51 2026 +0200

    #2090 Fetcher bolts: drop a set-header value coming from a response (#2168)
    
    The protocols read the headers to add to an outgoing request from
    <protocol.md.prefix>set-header. The fetcher bolts merge every response
    header into the tuple metadata under the same prefix, lowercased, so a
    server returning a header literally named set-header writes that key.
    Where the operator persists or transfers protocol. keys, which
    internals.adoc recommends for protocol.etag and protocol.set-cookie and
    which accepts wildcards, the forged value is carried to the next fetch
    and the crawled site chooses headers sent afterwards, including on
    requests to other hosts.
    
    The merge now drops the key the same way it already drops
    robots.crawl.delay, and puts back the value configured for the URL.
---
 .../org/apache/stormcrawler/bolt/FetcherBolt.java  | 10 +++
 .../stormcrawler/bolt/SimpleFetcherBolt.java       | 10 +++
 .../protocol/AbstractHttpProtocol.java             |  7 +-
 .../stormcrawler/bolt/AbstractFetcherBoltTest.java | 81 +++++++++++++++++++++-
 4 files changed, 105 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
index 6f6b0c9d..40666c25 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
@@ -58,6 +58,7 @@ import org.apache.stormcrawler.metrics.CrawlerMetrics;
 import org.apache.stormcrawler.metrics.ScopedCounter;
 import org.apache.stormcrawler.metrics.ScopedReducedMetric;
 import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.protocol.AbstractHttpProtocol;
 import org.apache.stormcrawler.protocol.FetchTimeoutException;
 import org.apache.stormcrawler.protocol.Protocol;
 import org.apache.stormcrawler.protocol.ProtocolFactory;
@@ -983,6 +984,15 @@ public class FetcherBolt extends StatusEmitterBolt {
                     // Only the locally parsed robots.txt value may populate this control signal.
                     // A colliding protocol prefix/header must not pace an unrelated queue.
                     mergedMetadata.remove(Constants.ROBOTS_CRAWL_DELAY_KEY);
+
+                    // Request shaping comes from the configuration, never from a fetched page: a
+                    // response header named set-header lands on exactly the key the protocols read
+                    // to add headers to an outgoing request. Any value configured for this URL is
+                    // put back after the response metadata has been dropped.
+                    final String setHeaderKey =
+                            protocolMetadataPrefix + AbstractHttpProtocol.SET_HEADER_BY_REQUEST;
+                    mergedMetadata.remove(setHeaderKey);
+                    mergedMetadata.setValues(setHeaderKey, metadata.getValues(setHeaderKey));
                     if (robotsCrawlDelaySecs != null) {
                         mergedMetadata.setValue(
                                 Constants.ROBOTS_CRAWL_DELAY_KEY, robotsCrawlDelaySecs);
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
index a7eda57b..a99dcc40 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
@@ -46,6 +46,7 @@ import org.apache.stormcrawler.metrics.CrawlerMetrics;
 import org.apache.stormcrawler.metrics.ScopedCounter;
 import org.apache.stormcrawler.metrics.ScopedReducedMetric;
 import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.protocol.AbstractHttpProtocol;
 import org.apache.stormcrawler.protocol.FetchTimeoutException;
 import org.apache.stormcrawler.protocol.Protocol;
 import org.apache.stormcrawler.protocol.ProtocolFactory;
@@ -495,6 +496,15 @@ public class SimpleFetcherBolt extends StatusEmitterBolt {
             // Only the locally parsed robots.txt value may populate this control signal.
             // A colliding protocol prefix/header must not pace an unrelated queue.
             mergedMetadata.remove(Constants.ROBOTS_CRAWL_DELAY_KEY);
+
+            // Request shaping comes from the configuration, never from a fetched page: a
+            // response header named set-header lands on exactly the key the protocols read
+            // to add headers to an outgoing request. Any value configured for this URL is
+            // put back after the response metadata has been dropped.
+            final String setHeaderKey =
+                    protocolMetadataPrefix + AbstractHttpProtocol.SET_HEADER_BY_REQUEST;
+            mergedMetadata.remove(setHeaderKey);
+            mergedMetadata.setValues(setHeaderKey, metadata.getValues(setHeaderKey));
             if (robotsCrawlDelaySecs != null) {
                 mergedMetadata.setValue(Constants.ROBOTS_CRAWL_DELAY_KEY, robotsCrawlDelaySecs);
             }
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/AbstractHttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/AbstractHttpProtocol.java
index 6f9b091f..dfb38261 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/AbstractHttpProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/AbstractHttpProtocol.java
@@ -77,7 +77,12 @@ public abstract class AbstractHttpProtocol implements Protocol {
      */
     protected static final String RESPONSE_COOKIES_ORIGIN = "set-cookie-origin";
 
-    protected static final String SET_HEADER_BY_REQUEST = "set-header";
+    /**
+     * Metadata key holding the headers to add to an outgoing request. It is a configuration
+     * setting: the fetcher bolts drop it from the metadata merged out of a response so that a
+     * crawled site cannot shape the requests sent afterwards.
+     */
+    public static final String SET_HEADER_BY_REQUEST = "set-header";
 
     protected String protocolMetadataPrefix = "";
 
diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java
index 8614b582..2a5ec832 100644
--- a/core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java
@@ -20,6 +20,7 @@ package org.apache.stormcrawler.bolt;
 import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
 import static com.github.tomakehurst.wiremock.client.WireMock.get;
 import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
 import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching;
 import static org.awaitility.Awaitility.await;
 import static org.mockito.Mockito.mock;
@@ -44,6 +45,7 @@ import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.TestOutputCollector;
 import org.apache.stormcrawler.TestUtil;
 import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.protocol.AbstractHttpProtocol;
 import org.apache.stormcrawler.protocol.ProtocolFactory;
 import org.apache.stormcrawler.protocol.StuckProtocol;
 import org.junit.jupiter.api.AfterEach;
@@ -53,6 +55,9 @@ import org.junit.jupiter.api.Test;
 @WireMockTest
 abstract class AbstractFetcherBoltTest {
 
+    /** Value of protocol.md.prefix in crawler-default.yaml, set explicitly by the tests. */
+    static final String PROTOCOL_MD_PREFIX = "protocol.";
+
     BaseRichBolt bolt;
 
     @AfterEach
@@ -361,9 +366,71 @@ abstract class AbstractFetcherBoltTest {
         Assertions.assertEquals(0, output.getEmitted(Utils.DEFAULT_STREAM_ID).size());
     }
 
+    /**
+     * A response header named set-header lands on the same metadata key the protocols read to add
+     * headers to an outgoing request, so a crawled site could shape the requests sent afterwards.
+     * The merge must drop it, see issue #2090.
+     */
+    @Test
+    void setHeaderFromResponseIsNotMerged(WireMockRuntimeInfo wmRuntimeInfo)
+            throws ReflectiveOperationException {
+        stubFor(
+                get(urlEqualTo("/set-header"))
+                        .willReturn(
+                                aResponse()
+                                        .withStatus(200)
+                                        .withHeader("set-header", "X-Injected=yes")
+                                        .withBody("hello")));
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("http.agent.name", "this_is_only_a_test");
+        config.put("protocol.md.prefix", PROTOCOL_MD_PREFIX);
+
+        Metadata md = fetchAndGetContentMetadata(wmRuntimeInfo, config, "/set-header");
+        Assertions.assertNull(
+                md.getFirstValue(AbstractHttpProtocol.SET_HEADER_BY_REQUEST, PROTOCOL_MD_PREFIX));
+    }
+
+    /** Dropping the response value must leave a value configured for the URL in place. */
+    @Test
+    void configuredSetHeaderSurvivesTheMerge(WireMockRuntimeInfo wmRuntimeInfo)
+            throws ReflectiveOperationException {
+        stubFor(
+                get(urlEqualTo("/set-header"))
+                        .willReturn(
+                                aResponse()
+                                        .withStatus(200)
+                                        .withHeader("set-header", "X-Injected=yes")
+                                        .withBody("hello")));
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("http.agent.name", "this_is_only_a_test");
+        config.put("protocol.md.prefix", PROTOCOL_MD_PREFIX);
+
+        Metadata sourceMetadata = new Metadata();
+        sourceMetadata.setValue(
+                PROTOCOL_MD_PREFIX + AbstractHttpProtocol.SET_HEADER_BY_REQUEST,
+                "X-Configured=yes");
+
+        Metadata md =
+                fetchAndGetContentMetadata(wmRuntimeInfo, config, "/set-header", sourceMetadata);
+        Assertions.assertArrayEquals(
+                new String[] {"X-Configured=yes"},
+                md.getValues(AbstractHttpProtocol.SET_HEADER_BY_REQUEST, PROTOCOL_MD_PREFIX));
+    }
+
     TestOutputCollector fetch(
             WireMockRuntimeInfo wmRuntimeInfo, Map<String, Object> config, String path)
             throws ReflectiveOperationException {
+        return fetch(wmRuntimeInfo, config, path, null);
+    }
+
+    TestOutputCollector fetch(
+            WireMockRuntimeInfo wmRuntimeInfo,
+            Map<String, Object> config,
+            String path,
+            Metadata sourceMetadata)
+            throws ReflectiveOperationException {
         resetProtocolFactory();
         TestOutputCollector output = new TestOutputCollector();
         bolt.prepare(config, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
@@ -372,7 +439,8 @@ abstract class AbstractFetcherBoltTest {
         when(tuple.getSourceComponent()).thenReturn("source");
         when(tuple.getStringByField("url"))
                 .thenReturn("http://localhost:" + wmRuntimeInfo.getHttpPort() + path);
-        when(tuple.getValueByField("metadata")).thenReturn(null);
+        when(tuple.contains("metadata")).thenReturn(sourceMetadata != null);
+        when(tuple.getValueByField("metadata")).thenReturn(sourceMetadata);
         bolt.execute(tuple);
 
         await().atMost(30, TimeUnit.SECONDS)
@@ -391,7 +459,16 @@ abstract class AbstractFetcherBoltTest {
     Metadata fetchAndGetContentMetadata(
             WireMockRuntimeInfo wmRuntimeInfo, Map<String, Object> config, String path)
             throws ReflectiveOperationException {
-        TestOutputCollector output = fetch(wmRuntimeInfo, config, path);
+        return fetchAndGetContentMetadata(wmRuntimeInfo, config, path, null);
+    }
+
+    Metadata fetchAndGetContentMetadata(
+            WireMockRuntimeInfo wmRuntimeInfo,
+            Map<String, Object> config,
+            String path,
+            Metadata sourceMetadata)
+            throws ReflectiveOperationException {
+        TestOutputCollector output = fetch(wmRuntimeInfo, config, path, sourceMetadata);
         List<List<Object>> contentTuples = output.getEmitted(Utils.DEFAULT_STREAM_ID);
         Assertions.assertEquals(1, contentTuples.size());
         Assertions.assertEquals(0, output.getEmitted(Constants.StatusStreamName).size());