You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by rz...@apache.org on 2026/09/13 06:53:45 UTC
(stormcrawler) branch main updated: Run redirect hops through the URL filters (#2128)
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
The following commit(s) were added to refs/heads/main by this push:
new 68b9ecbf Run redirect hops through the URL filters (#2128)
68b9ecbf is described below
commit 68b9ecbf17574b46584b003c36fa93cf8ac2ba3b
Author: Abhinav <al...@gmail.com>
AuthorDate: Sun Sep 13 12:23:40 2026 +0530
Run redirect hops through the URL filters (#2128)
* Run redirect hops through the URL filters (#2088)
Redirects are followed in HttpProtocol instead of by the client, so
every target passes through the URL filter chain before the hop is
taken. Credential headers are stripped when the host changes or the
target is not authenticated, reusing the credential policy from #2126.
The number of hops is limited by http.allow.redirects.max and the final
URL is recorded as _redirectedTo.
Rebased onto main after #2126 was merged.
* Strip credentials on any origin change during redirect hops (#2088)
Compare scheme, host and port instead of the host only, like OkHttp's own
redirect follower. Load the URL filters only when redirects are followed and
document http.allow.redirects.max.
---------
Co-authored-by: Richard Zowalla <rz...@apache.org>
---
.../stormcrawler/protocol/ProtocolResponse.java | 9 +
.../stormcrawler/protocol/okhttp/HttpProtocol.java | 208 ++++++++++++-
core/src/main/resources/crawler-default.yaml | 6 +
.../protocol/OkHttpFollowRedirectsTest.java | 321 +++++++++++++++++++++
.../protocol/okhttp/OkHttpTrustEverythingTest.java | 13 +-
core/src/test/resources/urlfilters-reject-all.json | 8 +
docs/src/main/asciidoc/configuration.adoc | 5 +-
docs/src/main/asciidoc/internals.adoc | 2 +-
8 files changed, 557 insertions(+), 15 deletions(-)
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java
index 566ba741..d898d246 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java
@@ -63,6 +63,15 @@ public class ProtocolResponse {
*/
public static final String TRIMMED_RESPONSE_REASON_KEY = "http.trimmed.reason";
+ /**
+ * Metadata key which holds the URL a followed redirect chain ended on, set when the content of
+ * a response does not come from the URL it was requested with. Related to, but distinct from,
+ * the {@code _redirTo} key the fetcher and parser bolts set on {@code Status.REDIRECTION}
+ * tuples: that one names the immediate target of a redirect which was NOT followed, this one
+ * names where an actually followed chain ended.
+ */
+ public static final String REDIRECTED_TO_KEY = "_redirectedTo";
+
/**
* @since 1.17
* @see <a href="https://github.com/apache/stormcrawler/issues/776">Issue 776</a>
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
index 59b1b2ae..cda61b47 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
@@ -76,6 +76,7 @@ import org.apache.http.cookie.Cookie;
import org.apache.storm.Config;
import org.apache.stormcrawler.Constants;
import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.URLFilters;
import org.apache.stormcrawler.protocol.AbstractHttpProtocol;
import org.apache.stormcrawler.protocol.IPFilterRules;
import org.apache.stormcrawler.protocol.ProtocolResponse;
@@ -91,6 +92,13 @@ public class HttpProtocol extends AbstractHttpProtocol {
private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(HttpProtocol.class);
+ /**
+ * Default maximum number of redirect hops followed when {@code http.allow.redirects} is
+ * enabled. A chain which does not end within this many hops returns its last redirect response,
+ * which the caller handles like it does when redirect following is off.
+ */
+ private static final int DEFAULT_MAX_REDIRECT_HOPS = 5;
+
private final MediaType json = MediaType.parse("application/json; charset=utf-8");
private OkHttpClient client;
@@ -102,6 +110,15 @@ public class HttpProtocol extends AbstractHttpProtocol {
/** Accept partially fetched content as trimmed content */
private boolean partialContentAsTrimmed = false;
+ /** Redirect targets run through the URL filters before a hop is taken. */
+ private URLFilters urlFilters = URLFilters.emptyURLFilters;
+
+ /** Whether redirect responses are followed, from {@code http.allow.redirects}. */
+ private boolean followRedirects = false;
+
+ /** Maximum number of redirect hops followed, from {@code http.allow.redirects.max}. */
+ private int maxRedirectHops = DEFAULT_MAX_REDIRECT_HOPS;
+
private final List<KeyValue> customRequestHeaders = new LinkedList<>();
// track the time spent for each URL in DNS resolution
@@ -226,12 +243,28 @@ public class HttpProtocol extends AbstractHttpProtocol {
+ "the host name either, the identity of the servers is not authenticated.");
}
+ /*
+ * Redirects are followed in getProtocolOutput and not by the client:
+ * every target has to pass through the URL filters first, so that the
+ * scheme exclusions, host confinement and depth rules configured for
+ * the crawl also apply to the hops a fetched page steers the fetcher
+ * to. The client must therefore never follow them on its own.
+ */
+ final boolean allowRedirects = ConfUtils.getBoolean(conf, "http.allow.redirects", false);
+ this.followRedirects = allowRedirects;
+ this.maxRedirectHops =
+ Math.max(
+ 1,
+ ConfUtils.getInt(
+ conf, "http.allow.redirects.max", DEFAULT_MAX_REDIRECT_HOPS));
+ // the filter chain is only needed to vet redirect hops
+ urlFilters = allowRedirects ? URLFilters.fromConf(conf) : URLFilters.emptyURLFilters;
builder =
new OkHttpClient.Builder()
.retryOnConnectionFailure(
ConfUtils.getBoolean(
conf, "http.retry.on.connection.failure", true))
- .followRedirects(ConfUtils.getBoolean(conf, "http.allow.redirects", false))
+ .followRedirects(false)
.connectTimeout(timeout, TimeUnit.MILLISECONDS)
.writeTimeout(timeout, TimeUnit.MILLISECONDS)
.readTimeout(timeout, TimeUnit.MILLISECONDS);
@@ -514,6 +547,18 @@ public class HttpProtocol extends AbstractHttpProtocol {
}
}
+ /** Removes every credential header except Proxy-Authorization (which is for the proxy). */
+ private void stripCredentialHeaders(Request.Builder builder) {
+ for (String name : new HashSet<>(builder.build().headers().names())) {
+ if (HttpHeaders.PROXY_AUTHORIZATION.equalsIgnoreCase(name)) {
+ continue;
+ }
+ if (isCredentialHeader(name)) {
+ builder.removeHeader(name);
+ }
+ }
+ }
+
@Override
public ProtocolResponse getProtocolOutput(String url, final Metadata metadata)
throws Exception {
@@ -657,9 +702,121 @@ public class HttpProtocol extends AbstractHttpProtocol {
final Request request = rb.build();
- final Call call = localClient.newCall(request);
+ /*
+ * Follow redirect responses manually: every target runs through the
+ * URL filters before the hop is taken, and a target which is rejected
+ * ends the chain - the redirect response is then returned as is and
+ * the caller handles it like it does when http.allow.redirects is
+ * off. The final URL is recorded in the response metadata so that
+ * callers can tell that the content is not from the URL they asked
+ * for.
+ */
+ Response lastResponse = null;
+ Call call = null;
+ Request currentRequest = request;
+ String currentUrl = url;
+ // the proxy is chosen per fetch above, so the chain has to use the
+ // same client the first request went out on
+ final OkHttpClient fetchClient = localClient;
+ // every hop creates its own Call and DNS timing entry: track them all
+ // so intermediate entries are cleaned up too, including on exceptions
+ final List<Call> hopCalls = new ArrayList<>();
+
+ try {
+ for (int hops = 0; hops <= maxRedirectHops; hops++) {
+ if (lastResponse != null) {
+ // release the connection before issuing the next request
+ lastResponse.close();
+ lastResponse = null;
+ }
+ call = fetchClient.newCall(currentRequest);
+ hopCalls.add(call);
+ try {
+ lastResponse = call.execute();
+ } catch (IOException | RuntimeException e) {
+ DNStimes.remove(call.toString());
+ throw e;
+ }
+
+ if (hops == maxRedirectHops) {
+ if (isRedirect(lastResponse)) {
+ LOG.warn("More than {} redirect hops for {}", maxRedirectHops, url);
+ }
+ break;
+ }
+ if (!followRedirects || !isRedirect(lastResponse)) {
+ break;
+ }
+
+ final String location = lastResponse.header(HttpHeaders.LOCATION);
+ if (StringUtils.isBlank(location)) {
+ LOG.debug(
+ "Got redirect response {} for {} without location",
+ lastResponse.code(),
+ url);
+ break;
+ }
+
+ final HttpUrl target = currentRequest.url().resolve(location);
+ if (target == null) {
+ LOG.warn(
+ "Redirect target {} could not be resolved against {}",
+ location,
+ currentUrl);
+ break;
+ }
+
+ final Metadata sourceMetadata = metadata != null ? metadata : new Metadata();
+ final String filtered =
+ urlFilters.filter(
+ currentRequest.url().url(), sourceMetadata, target.toString());
+ if (filtered == null) {
+ LOG.info("Redirect target {} rejected by the URL filters", target);
+ break;
+ }
+
+ final HttpUrl accepted = HttpUrl.parse(filtered);
+ if (accepted == null) {
+ LOG.warn("Filtered redirect target {} is not a URL", filtered);
+ break;
+ }
+
+ // an https to http downgrade is followed like the client does, the
+ // credential headers are stripped below unless
+ // http.credentials.allow.insecure is set
+ if (currentRequest.url().isHttps() && !accepted.isHttps()) {
+ LOG.warn("Redirect target {} downgrades https to http", accepted);
+ }
+
+ final Request.Builder followBuilder = currentRequest.newBuilder().url(accepted);
+ final HttpUrl from = currentRequest.url();
+ final boolean originChanged =
+ !from.scheme().equals(accepted.scheme())
+ || !from.host().equals(accepted.host())
+ || from.port() != accepted.port();
+ if (originChanged || !credentialsAllowed(accepted)) {
+ // a redirect must not hand the credentials of one origin
+ // (scheme, host and port) to another, even between two
+ // validated HTTPS origins, and must
+ // not send them to a server which is not authenticated
+ // (cleartext http://, trust-all or unverified host unless
+ // opted in)
+ stripCredentialHeaders(followBuilder);
+ }
+ // for 303, and for 301 or 302 after a POST, the request is
+ // repeated as a GET, like the client would do
+ final int code = lastResponse.code();
+ if (code == 303
+ || ((code == 301 || code == 302)
+ && !"GET".equals(currentRequest.method()))) {
+ followBuilder.method("GET", null);
+ }
+ currentRequest = followBuilder.build();
+ currentUrl = filtered;
+ }
- try (Response response = call.execute()) {
+ final Response response = lastResponse;
+ final Call executedCall = call;
final Metadata responsemetadata = new Metadata();
final Headers headers = response.headers();
@@ -678,6 +835,24 @@ public class HttpProtocol extends AbstractHttpProtocol {
responsemetadata.addValue(key.toLowerCase(Locale.ROOT), value);
}
+ if (isRedirect(response)) {
+ // a chain stopped after an intermediate hop returns a redirect
+ // whose Location is relative to the last request, not the
+ // original URL: resolve it there so callers (FetcherBolt)
+ // resolve the same target instead of one level too high
+ final String location = responsemetadata.getFirstValue("location");
+ if (StringUtils.isNotBlank(location)) {
+ final HttpUrl resolved = response.request().url().resolve(location);
+ if (resolved != null) {
+ responsemetadata.setValue("location", resolved.toString());
+ }
+ }
+ }
+
+ if (!currentUrl.equals(url)) {
+ responsemetadata.setValue(ProtocolResponse.REDIRECTED_TO_KEY, currentUrl);
+ }
+
// the Set-Cookie header does not say which host sent it: record the url of this
// response so that the cookies can be scoped to it when they are sent back. The
// key is dropped first so that a server sending a header of that name can not
@@ -692,8 +867,8 @@ public class HttpProtocol extends AbstractHttpProtocol {
new MutableObject<>(TrimmedContentReason.NOT_TRIMMED);
final byte[] bytes = toByteArray(response.body(), pageMaxContent, trimmed);
if (trimmed.get() != TrimmedContentReason.NOT_TRIMMED) {
- if (!call.isCanceled()) {
- call.cancel();
+ if (!executedCall.isCanceled()) {
+ executedCall.cancel();
}
responsemetadata.setValue(ProtocolResponse.TRIMMED_RESPONSE_KEY, "true");
responsemetadata.setValue(
@@ -702,15 +877,36 @@ public class HttpProtocol extends AbstractHttpProtocol {
LOG.warn("HTTP content trimmed to {} (reason: {})", bytes.length, trimmed.get());
}
- final Long dnsResolution = DNStimes.remove(call.toString());
+ final Long dnsResolution = DNStimes.remove(executedCall.toString());
if (dnsResolution != null) {
responsemetadata.setValue("metrics.dns.resolution.msec", dnsResolution.toString());
}
+ // drop intermediate hops' timing entries: only the final hop is reported
+ for (Call hopCall : hopCalls) {
+ if (hopCall != executedCall) {
+ DNStimes.remove(hopCall.toString());
+ }
+ }
return new ProtocolResponse(bytes, response.code(), responsemetadata);
+ } finally {
+ if (lastResponse != null) {
+ lastResponse.close();
+ }
+ // exception paths: no response to report from, but the hops'
+ // DNS entries must not accumulate over a long crawl
+ for (Call hopCall : hopCalls) {
+ DNStimes.remove(hopCall.toString());
+ }
}
}
+ /** Checks whether the response is a redirect whose Location must be resolved. */
+ private static boolean isRedirect(Response response) {
+ final int code = response.code();
+ return code == 301 || code == 302 || code == 303 || code == 307 || code == 308;
+ }
+
private byte[] toByteArray(
final ResponseBody responseBody,
int maxContent,
diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml
index e5f71732..1f3a0be2 100644
--- a/core/src/main/resources/crawler-default.yaml
+++ b/core/src/main/resources/crawler-default.yaml
@@ -188,6 +188,12 @@ config:
# http.credentials.headers:
# - "x-auth-token"
+ # Maximum number of redirect hops followed when http.allow.redirects is
+ # enabled (okhttp protocol only). A chain which does not end within this
+ # many hops returns its last redirect response, which the caller handles
+ # like it does when redirects are not followed at all.
+ http.allow.redirects.max: 5
+
# IP address filtering (okhttp protocol only). Optionally limit or block
# connections to IP address ranges once the host name has been resolved. This
# prevents information leakage to a public index when a DNS entry points to a
diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpFollowRedirectsTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpFollowRedirectsTest.java
new file mode 100644
index 00000000..942be408
--- /dev/null
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpFollowRedirectsTest.java
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.protocol;
+
+import jakarta.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import org.apache.storm.Config;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.protocol.okhttp.HttpProtocol;
+import org.eclipse.jetty.server.Handler;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.handler.AbstractHandler;
+import org.eclipse.jetty.server.handler.HandlerList;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/**
+ * When http.allow.redirects is enabled, every hop of the redirect chain runs through the URL
+ * filters, and the final URL is recorded in the response metadata.
+ */
+class OkHttpFollowRedirectsTest extends AbstractProtocolTest {
+
+ /**
+ * Redirects /start to /elsewhere (same origin), /cross to localhost (cross origin), and
+ * /nested/start through /nested/hop with a relative Location; serves plain text otherwise.
+ */
+ @Override
+ protected Handler[] getHandlers() {
+ return new Handler[] {
+ new AbstractHandler() {
+ @Override
+ public void handle(
+ String target,
+ Request baseRequest,
+ jakarta.servlet.http.HttpServletRequest request,
+ HttpServletResponse response)
+ throws IOException {
+ baseRequest.setHandled(true);
+ if (target.equals("/start")) {
+ response.setStatus(HttpServletResponse.SC_MOVED_TEMPORARILY);
+ response.setHeader(
+ "Location", "http://127.0.0.1:" + HTTP_PORT + "/elsewhere");
+ response.setContentLength(0);
+ response.getOutputStream().close();
+ return;
+ }
+ if (target.equals("/cross")) {
+ // same port but different host: another origin
+ response.setStatus(HttpServletResponse.SC_MOVED_TEMPORARILY);
+ response.setHeader(
+ "Location", "http://localhost:" + HTTP_PORT + "/elsewhere");
+ response.setContentLength(0);
+ response.getOutputStream().close();
+ return;
+ }
+ if (target.equals("/crossport")) {
+ // same host but different port: another origin
+ response.setStatus(HttpServletResponse.SC_MOVED_TEMPORARILY);
+ response.setHeader(
+ "Location", "http://127.0.0.1:" + otherPort + "/elsewhere");
+ response.setContentLength(0);
+ response.getOutputStream().close();
+ return;
+ }
+ if (target.equals("/nested/start")) {
+ response.setStatus(HttpServletResponse.SC_MOVED_TEMPORARILY);
+ response.setHeader(
+ "Location", "http://127.0.0.1:" + HTTP_PORT + "/nested/hop");
+ response.setContentLength(0);
+ response.getOutputStream().close();
+ return;
+ }
+ if (target.equals("/nested/hop")) {
+ // relative Location: resolves against /nested/hop, not /nested/start
+ response.setStatus(HttpServletResponse.SC_MOVED_TEMPORARILY);
+ response.setHeader("Location", "next");
+ response.setContentLength(0);
+ response.getOutputStream().close();
+ return;
+ }
+ if (target.equals("/elsewhere")) {
+ if (request.getHeader("Authorization") != null) {
+ seenSecondRequestHadAuthorization.set(true);
+ }
+ if (request.getHeader("X-Api-Key") != null) {
+ seenSecondRequestHadApiKey.set(true);
+ }
+ }
+ response.setStatus(HttpServletResponse.SC_OK);
+ response.setContentType("text/plain");
+ final byte[] content = ("body of " + target).getBytes(StandardCharsets.UTF_8);
+ response.setContentLength(content.length);
+ try (OutputStream out = response.getOutputStream()) {
+ out.write(content);
+ }
+ }
+ }
+ };
+ }
+
+ private static HttpProtocol protocol(Config conf) {
+ HttpProtocol protocol = new HttpProtocol();
+ protocol.configure(conf);
+ return protocol;
+ }
+
+ private static Config config() {
+ Config conf = new Config();
+ conf.put("http.agent.name", "this_is_only_a_test");
+ conf.put("http.allow.redirects", true);
+ return conf;
+ }
+
+ @Test
+ void redirectTargetIsFollowedAndRecorded() throws Exception {
+ // no urlfilters.config.file: the chain is empty, every target passes
+ HttpProtocol protocol = protocol(config());
+ ProtocolResponse response =
+ protocol.getProtocolOutput(
+ "http://127.0.0.1:" + HTTP_PORT + "/start", new Metadata());
+ protocol.cleanup();
+ Assertions.assertEquals(200, response.getStatusCode());
+ Assertions.assertEquals(
+ "body of /elsewhere",
+ new String(response.getContent(), StandardCharsets.UTF_8),
+ "the redirect is followed");
+ Assertions.assertEquals(
+ "http://127.0.0.1:" + HTTP_PORT + "/elsewhere",
+ response.getMetadata().getFirstValue(ProtocolResponse.REDIRECTED_TO_KEY),
+ "the final URL must be recorded in the response metadata");
+ }
+
+ @Test
+ void rejectedRedirectTargetIsNotFetched() throws Exception {
+ // a chain which rejects everything must stop the hop from being taken
+ Config conf = config();
+ conf.put("urlfilters.config.file", "urlfilters-reject-all.json");
+ HttpProtocol protocol = protocol(conf);
+ ProtocolResponse response =
+ protocol.getProtocolOutput(
+ "http://127.0.0.1:" + HTTP_PORT + "/start", new Metadata());
+ protocol.cleanup();
+ Assertions.assertEquals(
+ 302, response.getStatusCode(), "the redirect response itself is returned");
+ Assertions.assertEquals(
+ "http://127.0.0.1:" + HTTP_PORT + "/elsewhere",
+ response.getMetadata().getFirstValue("location"),
+ "the Location header tells the caller where the chain stopped");
+ }
+
+ /** Stands in for an exclusion rule which rejects every target. */
+ public static class RejectAllURLFilter extends org.apache.stormcrawler.filtering.URLFilter {
+ @Override
+ public String filter(
+ java.net.URL sourceUrl,
+ Metadata sourceMetadata,
+ @org.jetbrains.annotations.NotNull String urlToFilter) {
+ return null;
+ }
+ }
+
+ @Test
+ void redirectsAreNotFollowedWhenDisabled() throws Exception {
+ Config conf = config();
+ conf.put("http.allow.redirects", false);
+ HttpProtocol protocol = protocol(conf);
+ ProtocolResponse response =
+ protocol.getProtocolOutput(
+ "http://127.0.0.1:" + HTTP_PORT + "/start", new Metadata());
+ protocol.cleanup();
+ Assertions.assertEquals(
+ 302,
+ response.getStatusCode(),
+ "the redirect response itself is returned, nothing is followed");
+ Assertions.assertNull(
+ response.getMetadata().getFirstValue(ProtocolResponse.REDIRECTED_TO_KEY),
+ "no redirect was followed, so no final URL is recorded");
+ }
+
+ /** Same-origin hops keep every credential header, including custom ones. */
+ @Test
+ void credentialsSurviveASameOriginHop() throws Exception {
+ seenSecondRequestHadAuthorization.set(false);
+ seenSecondRequestHadApiKey.set(false);
+ Config conf = config();
+ // the test server is cleartext HTTP: opt in so the initial hop sends
+ conf.put("http.credentials.allow.insecure", true);
+ conf.put("http.basicauth.user", "user");
+ conf.put("http.basicauth.password", "secret");
+ conf.put("http.custom.headers", java.util.List.of("X-Api-Key=s3cret"));
+ HttpProtocol protocol = protocol(conf);
+ protocol.getProtocolOutput("http://127.0.0.1:" + HTTP_PORT + "/start", new Metadata());
+ protocol.cleanup();
+ Assertions.assertTrue(
+ seenSecondRequestHadAuthorization.get(),
+ "credentials must survive a same-origin redirect hop");
+ Assertions.assertTrue(
+ seenSecondRequestHadApiKey.get(),
+ "custom credential headers must survive a same-origin redirect hop");
+ }
+
+ /** A hop to another origin (different host) must not carry any credential header. */
+ @Test
+ void credentialsAreStrippedOnACrossOriginHop() throws Exception {
+ seenSecondRequestHadAuthorization.set(false);
+ seenSecondRequestHadApiKey.set(false);
+ Config conf = config();
+ // opt in so the initial hop sends: stripping then proves the origin
+ // change removes them even where the transport policy would allow them
+ conf.put("http.credentials.allow.insecure", true);
+ conf.put("http.basicauth.user", "user");
+ conf.put("http.basicauth.password", "secret");
+ conf.put("http.custom.headers", java.util.List.of("X-Api-Key=s3cret"));
+ HttpProtocol protocol = protocol(conf);
+ ProtocolResponse response =
+ protocol.getProtocolOutput(
+ "http://127.0.0.1:" + HTTP_PORT + "/cross", new Metadata());
+ protocol.cleanup();
+ Assertions.assertEquals(200, response.getStatusCode(), "the cross-origin hop is followed");
+ Assertions.assertFalse(
+ seenSecondRequestHadAuthorization.get(),
+ "Authorization must not reach the second origin");
+ Assertions.assertFalse(
+ seenSecondRequestHadApiKey.get(), "X-Api-Key must not reach the second origin");
+ }
+
+ /** A hop to the same host on another port is another origin and must not carry credentials. */
+ @Test
+ void credentialsAreStrippedOnACrossPortHop() throws Exception {
+ seenSecondRequestHadAuthorization.set(false);
+ seenSecondRequestHadApiKey.set(false);
+ final Server other = new Server(0);
+ final HandlerList handlers = new HandlerList();
+ handlers.setHandlers(getHandlers());
+ other.setHandler(handlers);
+ other.start();
+ try {
+ otherPort = ((ServerConnector) other.getConnectors()[0]).getLocalPort();
+ Config conf = config();
+ conf.put("http.credentials.allow.insecure", true);
+ conf.put("http.basicauth.user", "user");
+ conf.put("http.basicauth.password", "secret");
+ conf.put("http.custom.headers", java.util.List.of("X-Api-Key=s3cret"));
+ HttpProtocol protocol = protocol(conf);
+ ProtocolResponse response =
+ protocol.getProtocolOutput(
+ "http://127.0.0.1:" + HTTP_PORT + "/crossport", new Metadata());
+ protocol.cleanup();
+ Assertions.assertEquals(
+ 200, response.getStatusCode(), "the cross-port hop is followed");
+ Assertions.assertFalse(
+ seenSecondRequestHadAuthorization.get(),
+ "Authorization must not reach another port");
+ Assertions.assertFalse(
+ seenSecondRequestHadApiKey.get(), "X-Api-Key must not reach another port");
+ } finally {
+ other.stop();
+ }
+ }
+
+ /**
+ * When a chain stops after an intermediate hop, the returned Location is resolved against the
+ * last request URL: /start -> /nested/hop -> next with max 1 must yield /nested/next, not
+ * /next.
+ */
+ @Test
+ void relativeLocationResolvesAgainstLastHop() throws Exception {
+ Config conf = config();
+ conf.put("http.allow.redirects.max", 1);
+ HttpProtocol protocol = protocol(conf);
+ ProtocolResponse response =
+ protocol.getProtocolOutput(
+ "http://127.0.0.1:" + HTTP_PORT + "/nested/start", new Metadata());
+ protocol.cleanup();
+ Assertions.assertEquals(
+ 302, response.getStatusCode(), "the chain stops after the intermediate hop");
+ Assertions.assertEquals(
+ "http://127.0.0.1:" + HTTP_PORT + "/nested/next",
+ response.getMetadata().getFirstValue("location"),
+ "relative Location resolves against the last request URL");
+ }
+
+ /** Intermediate hops must not leave DNS timing entries behind. */
+ @Test
+ void dnsEntriesAreCleanedForEveryHop() throws Exception {
+ HttpProtocol protocol = protocol(config());
+ protocol.getProtocolOutput("http://127.0.0.1:" + HTTP_PORT + "/start", new Metadata());
+ java.lang.reflect.Field field = HttpProtocol.class.getDeclaredField("DNStimes");
+ field.setAccessible(true);
+ java.util.Map<?, ?> dnsTimes = (java.util.Map<?, ?>) field.get(protocol);
+ protocol.cleanup();
+ Assertions.assertTrue(
+ dnsTimes.isEmpty(), "every hop's DNS entry is cleaned up, not just the final one");
+ }
+
+ static volatile int otherPort;
+
+ static final java.util.concurrent.atomic.AtomicBoolean seenSecondRequestHadAuthorization =
+ new java.util.concurrent.atomic.AtomicBoolean(false);
+ static final java.util.concurrent.atomic.AtomicBoolean seenSecondRequestHadApiKey =
+ new java.util.concurrent.atomic.AtomicBoolean(false);
+}
diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java
index 814ec6f9..9dd9c586 100644
--- a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java
@@ -430,12 +430,12 @@ class OkHttpTrustEverythingTest {
}
/**
- * With the opt-in, the same redirect forwards custom credential headers to the HTTP hop.
- * (OkHttp itself strips Authorization on a scheme downgrade even then, which is its safe
- * default; the policy opt-in covers the headers StormCrawler controls.)
+ * The opt-in covers the transport of the URL the crawl targeted, not redirect hops: an https to
+ * http redirect changes the origin (scheme and port), so credential headers are stripped even
+ * then, like OkHttp's own redirect follower does.
*/
@Test
- void credentialsAreForwardedOnHttpsToHttpRedirectWhenInsecureAllowed() throws Exception {
+ void credentialsAreStrippedOnHttpsToHttpRedirectEvenWhenInsecureAllowed() throws Exception {
final Config conf = config();
conf.put("http.allow.redirects", true);
conf.put("http.credentials.allow.insecure", true);
@@ -456,10 +456,11 @@ class OkHttpTrustEverythingTest {
protocol,
"https://localhost:" + server.httpsPort() + "/redirect-allowed",
new Metadata());
+ server.verify(
+ 1, getRequestedFor(urlPathEqualTo("/target-allowed")).withoutHeader("X-Api-Key"));
server.verify(
1,
- getRequestedFor(urlPathEqualTo("/target-allowed"))
- .withHeader("X-Api-Key", equalTo("s3cret")));
+ getRequestedFor(urlPathEqualTo("/target-allowed")).withoutHeader("Authorization"));
}
/**
diff --git a/core/src/test/resources/urlfilters-reject-all.json b/core/src/test/resources/urlfilters-reject-all.json
new file mode 100644
index 00000000..c8169f4b
--- /dev/null
+++ b/core/src/test/resources/urlfilters-reject-all.json
@@ -0,0 +1,8 @@
+{
+ "org.apache.stormcrawler.filtering.URLFilters": [
+ {
+ "class": "org.apache.stormcrawler.protocol.OkHttpFollowRedirectsTest$RejectAllURLFilter",
+ "name": "RejectAllURLFilter"
+ }
+ ]
+}
diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc
index cc940a8b..093f60e5 100644
--- a/docs/src/main/asciidoc/configuration.adoc
+++ b/docs/src/main/asciidoc/configuration.adoc
@@ -232,7 +232,8 @@ implementation.
| partition.url.mode | byHost | Defines how URLs are partitioned: byHost, byDomain, or byIP.
| protocols | http,https | Supported protocols. The `file` scheme is not enabled by default; a crawl of a local corpus has to add it explicitly and confine reads with `file.protocol.root`.
| redirections.allowed | true | If true, emit redirect target URLs as "outlinks" to the status stream. If false, do not follow redirects. See also `http.allow.redirects`.
-| http.allow.redirects | false | (OkHttp only) Follow HTTP redirects immediately in the HTTP protocol client. Note: if followed immediately, redirect target URLs are not emitted to the status stream, are not filtered, not deduplicated, and not checked against robots.txt.
+| http.allow.redirects | false | (OkHttp only) Follow HTTP redirects immediately in the HTTP protocol client. Note: if followed immediately, redirect target URLs are not emitted to the status stream, are not deduplicated and not checked against robots.txt, but each target runs through the URL filter chain before the hop is taken (up to `http.allow.redirects.max` hops; the final URL is recorded in the response metadata as `_redirectedTo`; credentials are not carried to another origin or t [...]
+| http.allow.redirects.max | 5 | (OkHttp only) Maximum number of redirect hops followed when `http.allow.redirects` is enabled. A chain which does not end within this many hops returns its last redirect response, which is handled like a redirect that is not followed immediately.
| sitemap.discovery | false | Enable automatic sitemap discovery.
| urlbuffer.class | org.apache.stormcrawler.persistence.urlbuffer.SimpleURLBuffer | URL buffer implementation used by spouts.
|===
@@ -263,7 +264,7 @@ header.
| http.filter.ipaddress.exclude | localhost, sitelocal, linklocal, anylocal, multicast, 100.64.0.0/10, 0.0.0.0/8, fc00::/7, ::/128 | (OkHttp only) Comma-separated list (or YAML list) of blocked IP ranges. Set to `""` to crawl private address space. See <<IP Address Filtering>>.
| http.trust.everything | false | (OkHttp only) If true, accept any TLS certificate, including self-signed, expired or otherwise invalid ones. The servers are then not authenticated: anyone able to answer for the host name receives everything sent to them. Credentials (basic auth, credential headers, cookies) are withheld from cleartext http:// requests and from https:// requests on such connections unless `http.credentials.allow.insecure` is set to true.
| http.verify.hostnames | true | (OkHttp only) If true, check that the certificate presented by the server matches the host name contacted. Independent of `http.trust.everything`: a valid certificate for a different name is accepted when this is false. Credentials are then withheld unless `http.credentials.allow.insecure` is set to true.
-| http.credentials.allow.insecure | false | (OkHttp only) If true, send credentials (basic auth, credential headers, cookies) even on connections which do not authenticate the server: cleartext http:// urls and, when `http.trust.everything` is true or `http.verify.hostnames` is false, https:// urls as well.
+| http.credentials.allow.insecure | false | (OkHttp only) If true, send credentials (basic auth, credential headers, cookies) even on connections which do not authenticate the server: cleartext http:// urls and, when `http.trust.everything` is true or `http.verify.hostnames` is false, https:// urls as well. Redirect hops to another origin (scheme, host or port) never carry them.
| http.credentials.headers | authorization, proxy-authorization, cookie, x-api-key | (OkHttp only) Header names (case-insensitive) which carry credentials and are therefore withheld from requests to servers which were not authenticated. Applies to the headers configured with `http.custom.headers` and to headers set by request. Names given here are added to the four defaults, which are always treated as credentials.
| topology.message.timeout.secs | -1 | OKHTTP message timeout.
|===
diff --git a/docs/src/main/asciidoc/internals.adoc b/docs/src/main/asciidoc/internals.adoc
index e8ec8804..e98fd99d 100644
--- a/docs/src/main/asciidoc/internals.adoc
+++ b/docs/src/main/asciidoc/internals.adoc
@@ -387,7 +387,7 @@ When handling HTTP redirects, StormCrawler offers three modes:
2. **Redirects disabled** (`redirections.allowed: false`): Redirect target URLs are not sent to the status stream. Redirects are effectively ignored.
-3. **Immediate follow** (`http.allow.redirects: true`): Redirects are followed immediately in the HTTP client and the target URLs are not emitted to the status stream. This is the default behavior for the browser-based Playwright protocol, but it is also supported by the OkHttp protocol. Note that with immediate follows, redirect targets bypass URL filtering, deduplication, and `robots.txt` checks.
+3. **Immediate follow** (`http.allow.redirects: true`): Redirects are followed immediately by the protocol and the target URLs are not emitted to the status stream. This is the default behavior for the browser-based Playwright protocol, but it is also supported by the OkHttp protocol. With immediate follows on OkHttp, every redirect target runs through the URL filter chain before the hop is taken and a rejected target ends the chain, so the filter rules configured for the crawl still app [...]
==== Network Protocols