You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by jn...@apache.org on 2026/06/13 06:59:49 UTC
(stormcrawler) branch main updated: 1143 Add per-URL proxy metadata selection (#1935)
This is an automated email from the ASF dual-hosted git repository.
jnioche pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
The following commit(s) were added to refs/heads/main by this push:
new aab3e18b 1143 Add per-URL proxy metadata selection (#1935)
aab3e18b is described below
commit aab3e18b37430a7d08058a0b8e0c30845697194a
Author: preko-p <pr...@gmail.com>
AuthorDate: Sat Jun 13 08:59:44 2026 +0200
1143 Add per-URL proxy metadata selection (#1935)
* 1143 Add per-URL proxy metadata selection
Reference #1143.
AI-assistant: Codex CLI
* Address proxy metadata review feedback
---------
Co-authored-by: preko06 <is...@workl.com>
Co-authored-by: preko-p <27...@users.noreply.github.com>
---
.../stormcrawler/proxy/MultiProxyManager.java | 20 ++
.../apache/stormcrawler/proxy/ProxyMetadata.java | 148 ++++++++++++
.../org/apache/stormcrawler/proxy/ProxyUtils.java | 55 +++++
.../stormcrawler/proxy/SingleProxyManager.java | 44 +++-
.../stormcrawler/bolt/AbstractFetcherBoltTest.java | 47 ++++
.../HttpClientProtocolProxyManagerTest.java | 69 ++++++
.../okhttp/HttpProtocolProxyConcurrencyTest.java | 50 +++-
.../stormcrawler/proxy/MultiProxyManagerTest.java | 171 ++++++++++++++
.../stormcrawler/proxy/SingleProxyManagerTest.java | 263 +++++++++++++++++++++
docs/src/main/asciidoc/configuration.adoc | 26 +-
docs/src/main/asciidoc/internals.adoc | 8 +-
11 files changed, 874 insertions(+), 27 deletions(-)
diff --git a/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java b/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java
index 21eaf977..7140388a 100644
--- a/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java
+++ b/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java
@@ -214,8 +214,28 @@ public class MultiProxyManager implements ProxyManager {
return this.proxies.length;
}
+ private Optional<SCProxy> getConfiguredProxy(SCProxy proxy) {
+ for (SCProxy configuredProxy : this.proxies) {
+ if (ProxyUtils.isSameProxy(configuredProxy, proxy)) {
+ return Optional.of(configuredProxy);
+ }
+ }
+ return Optional.empty();
+ }
+
@Override
public Optional<SCProxy> getProxy(Metadata metadata) {
+ if (ProxyMetadata.shouldSkipProxy(metadata)) {
+ return Optional.empty();
+ }
+
+ Optional<SCProxy> metadataProxy = ProxyMetadata.getProxy(metadata);
+ if (metadataProxy.isPresent()) {
+ SCProxy proxy = getConfiguredProxy(metadataProxy.get()).orElse(metadataProxy.get());
+ proxy.incrementUsage();
+ return Optional.of(proxy);
+ }
+
// create a variable to hold the proxy generated in the following switch statement
SCProxy proxy;
diff --git a/core/src/main/java/org/apache/stormcrawler/proxy/ProxyMetadata.java b/core/src/main/java/org/apache/stormcrawler/proxy/ProxyMetadata.java
new file mode 100644
index 00000000..63e6761a
--- /dev/null
+++ b/core/src/main/java/org/apache/stormcrawler/proxy/ProxyMetadata.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.proxy;
+
+import java.util.Locale;
+import java.util.Optional;
+import org.apache.stormcrawler.Metadata;
+
+/** Utilities for resolving per-URL proxy settings from metadata. */
+final class ProxyMetadata {
+
+ static final String PROXY = "http.proxy";
+ static final String PROXY_HOST = "http.proxy.host";
+ static final String PROXY_PORT = "http.proxy.port";
+ static final String PROXY_TYPE = "http.proxy.type";
+ static final String PROXY_USER = "http.proxy.user";
+ static final String PROXY_PASS = "http.proxy.pass";
+ static final String PROXY_SKIP = "http.proxy.skip";
+
+ private ProxyMetadata() {}
+
+ static boolean shouldSkipProxy(Metadata metadata) {
+ if (metadata == null || !metadata.containsKey(PROXY_SKIP)) {
+ return false;
+ }
+
+ String skip = requiredValue(metadata, PROXY_SKIP);
+ if ("true".equalsIgnoreCase(skip)) {
+ return true;
+ }
+ if ("false".equalsIgnoreCase(skip)) {
+ return false;
+ }
+
+ throw new IllegalArgumentException(
+ "metadata key `" + PROXY_SKIP + "` must be `true` or `false`, got `" + skip + "`");
+ }
+
+ static Optional<SCProxy> getProxy(Metadata metadata) {
+ if (metadata == null) {
+ return Optional.empty();
+ }
+
+ if (metadata.containsKey(PROXY)) {
+ String proxy = requiredValue(metadata, PROXY);
+ try {
+ return Optional.of(new SCProxy(proxy));
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException(
+ "metadata key `" + PROXY + "` must be a valid proxy connection string");
+ }
+ }
+
+ if (!containsProxyField(metadata)) {
+ return Optional.empty();
+ }
+
+ String host = requiredValue(metadata, PROXY_HOST);
+ String type = valueOrDefault(metadata, PROXY_TYPE, "HTTP");
+ String port = valueOrDefault(metadata, PROXY_PORT, "8080");
+ String username = value(metadata, PROXY_USER);
+ String password = value(metadata, PROXY_PASS);
+
+ validatePort(port);
+ if ((username == null) != (password == null)) {
+ throw new IllegalArgumentException(
+ "metadata proxy authentication requires both `"
+ + PROXY_USER
+ + "` and `"
+ + PROXY_PASS
+ + "`");
+ }
+
+ return Optional.of(
+ new SCProxy(
+ type.toLowerCase(Locale.ROOT),
+ host,
+ port,
+ username == null ? "" : username,
+ password == null ? "" : password,
+ "",
+ "",
+ "",
+ ""));
+ }
+
+ private static boolean containsProxyField(Metadata metadata) {
+ return metadata.containsKey(PROXY_HOST)
+ || metadata.containsKey(PROXY_PORT)
+ || metadata.containsKey(PROXY_TYPE)
+ || metadata.containsKey(PROXY_USER)
+ || metadata.containsKey(PROXY_PASS);
+ }
+
+ private static String valueOrDefault(Metadata metadata, String key, String defaultValue) {
+ if (!metadata.containsKey(key)) {
+ return defaultValue;
+ }
+ return requiredValue(metadata, key);
+ }
+
+ private static String requiredValue(Metadata metadata, String key) {
+ String value = value(metadata, key);
+ if (value == null) {
+ throw new IllegalArgumentException("metadata key `" + key + "` must not be blank");
+ }
+ return value;
+ }
+
+ private static String value(Metadata metadata, String key) {
+ if (!metadata.containsKey(key)) {
+ return null;
+ }
+
+ String value = metadata.getFirstValue(key);
+ if (value == null || value.trim().isEmpty()) {
+ return null;
+ }
+ return value.trim();
+ }
+
+ private static void validatePort(String port) {
+ int parsedPort;
+ try {
+ parsedPort = Integer.parseInt(port);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(
+ "metadata key `" + PROXY_PORT + "` must be an integer, got `" + port + "`", e);
+ }
+
+ ProxyUtils.validatePortRange(parsedPort, "metadata key `" + PROXY_PORT + "`", port);
+ }
+}
diff --git a/core/src/main/java/org/apache/stormcrawler/proxy/ProxyUtils.java b/core/src/main/java/org/apache/stormcrawler/proxy/ProxyUtils.java
new file mode 100644
index 00000000..9d3647d4
--- /dev/null
+++ b/core/src/main/java/org/apache/stormcrawler/proxy/ProxyUtils.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.proxy;
+
+import java.util.Locale;
+import java.util.Objects;
+
+final class ProxyUtils {
+
+ static final int MIN_PORT = 1;
+ static final int MAX_PORT = 65535;
+
+ private ProxyUtils() {}
+
+ static void validatePortRange(int port, String source, String value) {
+ if (port < MIN_PORT || port > MAX_PORT) {
+ throw new IllegalArgumentException(
+ source
+ + " must be between "
+ + MIN_PORT
+ + " and "
+ + MAX_PORT
+ + ", got `"
+ + value
+ + "`");
+ }
+ }
+
+ static boolean isSameProxy(SCProxy proxy, SCProxy otherProxy) {
+ return Objects.equals(normalize(proxy.getProtocol()), normalize(otherProxy.getProtocol()))
+ && Objects.equals(normalize(proxy.getAddress()), normalize(otherProxy.getAddress()))
+ && Objects.equals(proxy.getPort(), otherProxy.getPort())
+ && Objects.equals(proxy.getUsername(), otherProxy.getUsername())
+ && Objects.equals(proxy.getPassword(), otherProxy.getPassword());
+ }
+
+ private static String normalize(String value) {
+ return value == null ? null : value.toLowerCase(Locale.ROOT);
+ }
+}
diff --git a/core/src/main/java/org/apache/stormcrawler/proxy/SingleProxyManager.java b/core/src/main/java/org/apache/stormcrawler/proxy/SingleProxyManager.java
index 078fa4d0..a55a10e0 100644
--- a/core/src/main/java/org/apache/stormcrawler/proxy/SingleProxyManager.java
+++ b/core/src/main/java/org/apache/stormcrawler/proxy/SingleProxyManager.java
@@ -39,30 +39,48 @@ public class SingleProxyManager implements ProxyManager {
// values for single proxy
String proxyHost = ConfUtils.getString(conf, "http.proxy.host", null);
+ if (proxyHost == null) {
+ this.proxy = null;
+ return;
+ }
+
String proxyType = ConfUtils.getString(conf, "http.proxy.type", "HTTP");
int proxyPort = ConfUtils.getInt(conf, "http.proxy.port", 8080);
String proxyUsername = ConfUtils.getString(conf, "http.proxy.user", null);
String proxyPassword = ConfUtils.getString(conf, "http.proxy.pass", null);
+ ProxyUtils.validatePortRange(
+ proxyPort, "config key `http.proxy.port`", Integer.toString(proxyPort));
- // assemble proxy connection string
- String proxyString = proxyType.toLowerCase(Locale.ROOT) + "://";
-
- // conditionally append authentication info
- if (proxyUsername != null
- && !proxyUsername.isEmpty()
- && proxyPassword != null
- && !proxyPassword.isEmpty()) {
- proxyString += proxyUsername + ":" + proxyPassword + "@";
- }
+ boolean hasAuth =
+ proxyUsername != null
+ && !proxyUsername.isEmpty()
+ && proxyPassword != null
+ && !proxyPassword.isEmpty();
- // complete proxy string and create proxy
this.proxy =
new SCProxy(
- proxyString + String.format(Locale.ROOT, "%s:%d", proxyHost, proxyPort));
+ proxyType.toLowerCase(Locale.ROOT),
+ proxyHost,
+ Integer.toString(proxyPort),
+ hasAuth ? proxyUsername : "",
+ hasAuth ? proxyPassword : "",
+ "",
+ "",
+ "",
+ "");
}
@Override
public Optional<SCProxy> getProxy(Metadata metadata) {
- return Optional.of(proxy);
+ if (ProxyMetadata.shouldSkipProxy(metadata)) {
+ return Optional.empty();
+ }
+
+ Optional<SCProxy> metadataProxy = ProxyMetadata.getProxy(metadata);
+ if (metadataProxy.isPresent()) {
+ return metadataProxy;
+ }
+
+ return Optional.ofNullable(proxy);
}
}
diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java
index 8fc4d902..4f4aa521 100644
--- a/core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/AbstractFetcherBoltTest.java
@@ -28,6 +28,7 @@ import static org.mockito.Mockito.when;
import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
import com.github.tomakehurst.wiremock.junit5.WireMockTest;
import java.io.IOException;
+import java.lang.reflect.Field;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -41,6 +42,7 @@ import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.TestOutputCollector;
import org.apache.stormcrawler.TestUtil;
import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.protocol.ProtocolFactory;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -150,4 +152,49 @@ abstract class AbstractFetcherBoltTest {
// nothing on the default stream — no content was fetched
Assertions.assertEquals(0, output.getEmitted(Utils.DEFAULT_STREAM_ID).size());
}
+
+ @Test
+ void invalidProxyMetadataEmitsFetchError(WireMockRuntimeInfo wmRuntimeInfo)
+ throws ReflectiveOperationException {
+ stubFor(get(urlMatching("/invalid-proxy")).willReturn(aResponse().withStatus(200)));
+
+ resetProtocolFactory();
+ TestOutputCollector output = new TestOutputCollector();
+ Map<String, Object> config = new HashMap<>();
+ config.put("http.agent.name", "this_is_only_a_test");
+ config.put("http.proxy.manager", "org.apache.stormcrawler.proxy.SingleProxyManager");
+ bolt.prepare(config, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy.host", "proxy.example.com");
+ metadata.setValue("http.proxy.port", "not-a-port");
+
+ Tuple tuple = mock(Tuple.class);
+ String url = "http://localhost:" + wmRuntimeInfo.getHttpPort() + "/invalid-proxy";
+ when(tuple.getSourceComponent()).thenReturn("source");
+ when(tuple.contains("metadata")).thenReturn(true);
+ when(tuple.getStringByField("url")).thenReturn(url);
+ when(tuple.getValueByField("metadata")).thenReturn(metadata);
+ bolt.execute(tuple);
+
+ await().atMost(8, TimeUnit.SECONDS).until(() -> output.getAckedTuples().contains(tuple));
+
+ Assertions.assertFalse(output.getFailedTuples().contains(tuple));
+ List<List<Object>> statusTuples = output.getEmitted(Constants.StatusStreamName);
+ Assertions.assertEquals(1, statusTuples.size());
+ Assertions.assertEquals(url, statusTuples.get(0).get(0));
+ Metadata statusMetadata = (Metadata) statusTuples.get(0).get(1);
+ Assertions.assertEquals(Status.FETCH_ERROR, statusTuples.get(0).get(2));
+ Assertions.assertEquals(
+ IllegalArgumentException.class.getName(),
+ statusMetadata.getFirstValue("fetch.exception"));
+
+ Assertions.assertEquals(0, output.getEmitted(Utils.DEFAULT_STREAM_ID).size());
+ }
+
+ private void resetProtocolFactory() throws ReflectiveOperationException {
+ Field instance = ProtocolFactory.class.getDeclaredField("single_instance");
+ instance.setAccessible(true);
+ instance.set(null, null);
+ }
}
diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java
new file mode 100644
index 00000000..d67de936
--- /dev/null
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.protocol.httpclient;
+
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.storm.Config;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.proxy.SingleProxyManager;
+import org.junit.jupiter.api.Test;
+
+class HttpClientProtocolProxyManagerTest {
+
+ @Test
+ void omittedProxyManagerWithoutProxyHostDoesNotCreateDefaultManager() {
+ HttpProtocol protocol = new HttpProtocol();
+ protocol.configure(protocolConfig());
+
+ assertNull(protocol.proxyManager);
+ }
+
+ @Test
+ void omittedProxyManagerWithProxyHostCreatesSingleProxyManager() {
+ HttpProtocol protocol = new HttpProtocol();
+ Config conf = protocolConfig();
+ conf.put("http.proxy.host", "proxy.example.com");
+ protocol.configure(conf);
+
+ assertInstanceOf(SingleProxyManager.class, protocol.proxyManager);
+ }
+
+ @Test
+ void explicitSingleProxyManagerWithoutDefaultProxySupportsMetadataOnlyUrls() {
+ HttpProtocol protocol = new HttpProtocol();
+ Config conf = protocolConfig();
+ conf.put("http.proxy.manager", "org.apache.stormcrawler.proxy.SingleProxyManager");
+ protocol.configure(conf);
+
+ assertInstanceOf(SingleProxyManager.class, protocol.proxyManager);
+ assertTrue(protocol.proxyManager.getProxy(new Metadata()).isEmpty());
+ }
+
+ private Config protocolConfig() {
+ Config conf = new Config();
+ conf.put("http.agent.name", "test");
+ conf.put("http.agent.version", "1.0");
+ conf.put("http.agent.description", "test");
+ conf.put("http.agent.url", "http://test.example.com");
+ conf.put("http.agent.email", "test@example.com");
+ return conf;
+ }
+}
diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolProxyConcurrencyTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolProxyConcurrencyTest.java
index d6594fec..76753ff0 100644
--- a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolProxyConcurrencyTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolProxyConcurrencyTest.java
@@ -17,7 +17,9 @@
package org.apache.stormcrawler.protocol.okhttp;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.Optional;
import okhttp3.OkHttpClient;
@@ -26,6 +28,7 @@ import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.protocol.AbstractProtocolTest;
import org.apache.stormcrawler.proxy.ProxyManager;
import org.apache.stormcrawler.proxy.SCProxy;
+import org.apache.stormcrawler.proxy.SingleProxyManager;
import org.junit.jupiter.api.Test;
/**
@@ -42,6 +45,35 @@ import org.junit.jupiter.api.Test;
*/
class HttpProtocolProxyConcurrencyTest extends AbstractProtocolTest {
+ @Test
+ void omittedProxyManagerWithoutProxyHostDoesNotCreateDefaultManager() {
+ HttpProtocol protocol = new HttpProtocol();
+ protocol.configure(protocolConfig());
+
+ assertNull(protocol.proxyManager);
+ }
+
+ @Test
+ void omittedProxyManagerWithProxyHostCreatesSingleProxyManager() {
+ HttpProtocol protocol = new HttpProtocol();
+ Config conf = protocolConfig();
+ conf.put("http.proxy.host", "proxy.example.com");
+ protocol.configure(conf);
+
+ assertInstanceOf(SingleProxyManager.class, protocol.proxyManager);
+ }
+
+ @Test
+ void explicitSingleProxyManagerWithoutDefaultProxySupportsMetadataOnlyUrls() {
+ HttpProtocol protocol = new HttpProtocol();
+ Config conf = protocolConfig();
+ conf.put("http.proxy.manager", "org.apache.stormcrawler.proxy.SingleProxyManager");
+ protocol.configure(conf);
+
+ assertInstanceOf(SingleProxyManager.class, protocol.proxyManager);
+ assertTrue(protocol.proxyManager.getProxy(new Metadata()).isEmpty());
+ }
+
/**
* After a proxied request completes, the shared builder should not retain any proxy
* configuration. With the old buggy code, {@code builder.proxy(proxy)} permanently set the
@@ -50,13 +82,7 @@ class HttpProtocolProxyConcurrencyTest extends AbstractProtocolTest {
@Test
void proxiedRequestShouldNotPolluteSharedBuilder() throws Exception {
HttpProtocol protocol = new HttpProtocol();
- Config conf = new Config();
- conf.put("http.agent.name", "test");
- conf.put("http.agent.version", "1.0");
- conf.put("http.agent.description", "test");
- conf.put("http.agent.url", "http://test.example.com");
- conf.put("http.agent.email", "test@example.com");
- protocol.configure(conf);
+ protocol.configure(protocolConfig());
protocol.proxyManager =
new ProxyManager() {
@@ -93,4 +119,14 @@ class HttpProtocolProxyConcurrencyTest extends AbstractProtocolTest {
+ "This indicates the bug where getProtocolOutput() mutates the shared "
+ "builder instead of using a per-request builder.");
}
+
+ private Config protocolConfig() {
+ Config conf = new Config();
+ conf.put("http.agent.name", "test");
+ conf.put("http.agent.version", "1.0");
+ conf.put("http.agent.description", "test");
+ conf.put("http.agent.url", "http://test.example.com");
+ conf.put("http.agent.email", "test@example.com");
+ return conf;
+ }
}
diff --git a/core/src/test/java/org/apache/stormcrawler/proxy/MultiProxyManagerTest.java b/core/src/test/java/org/apache/stormcrawler/proxy/MultiProxyManagerTest.java
index c1cd3287..7b988567 100644
--- a/core/src/test/java/org/apache/stormcrawler/proxy/MultiProxyManagerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/proxy/MultiProxyManagerTest.java
@@ -25,6 +25,7 @@ import java.nio.file.Paths;
import java.util.Optional;
import java.util.function.Supplier;
import org.apache.storm.Config;
+import org.apache.stormcrawler.Metadata;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -173,6 +174,176 @@ class MultiProxyManagerTest {
Assertions.assertEquals(proxy3.toString(), proxy6.toString());
}
+ @Test
+ void metadataSkipDoesNotAdvanceRoundRobinRotation() {
+ String[] proxyStrings = {
+ "http://first.example.com:8080", "http://second.example.com:8080",
+ };
+ MultiProxyManager pm = new MultiProxyManager();
+ pm.configure(MultiProxyManager.ProxyRotation.ROUND_ROBIN, proxyStrings);
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy.skip", "true");
+ metadata.setValue("http.proxy", "http://metadata.example.com:8080");
+
+ Assertions.assertTrue(pm.getProxy(metadata).isEmpty());
+ Assertions.assertEquals(
+ "http://first.example.com:8080", pm.getProxy(null).get().toString());
+ }
+
+ @Test
+ void metadataSkipFalseUsesRoundRobinRotation() {
+ String[] proxyStrings = {
+ "http://first.example.com:8080", "http://second.example.com:8080",
+ };
+ MultiProxyManager pm = new MultiProxyManager();
+ pm.configure(MultiProxyManager.ProxyRotation.ROUND_ROBIN, proxyStrings);
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy.skip", "false");
+
+ Assertions.assertEquals(
+ "http://first.example.com:8080", pm.getProxy(metadata).get().toString());
+ }
+
+ @Test
+ void invalidMetadataSkipFailsFast() {
+ String[] proxyStrings = {
+ "http://first.example.com:8080", "http://second.example.com:8080",
+ };
+ MultiProxyManager pm = new MultiProxyManager();
+ pm.configure(MultiProxyManager.ProxyRotation.ROUND_ROBIN, proxyStrings);
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy.skip", "not-a-boolean");
+
+ Assertions.assertThrows(IllegalArgumentException.class, () -> pm.getProxy(metadata));
+ }
+
+ @Test
+ void metadataOverrideUsesConfiguredProxyInstanceWhenMatched() {
+ String[] proxyStrings = {
+ "http://first.example.com:8080", "http://second.example.com:8080",
+ };
+ MultiProxyManager pm = new MultiProxyManager();
+ pm.configure(MultiProxyManager.ProxyRotation.LEAST_USED, proxyStrings);
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy", "http://second.example.com:8080");
+
+ SCProxy metadataProxy = pm.getProxy(metadata).get();
+ Assertions.assertEquals("http://second.example.com:8080", metadataProxy.toString());
+ Assertions.assertEquals(1, metadataProxy.getUsage());
+
+ SCProxy nextProxy = pm.getProxy(null).get();
+ Assertions.assertEquals("http://first.example.com:8080", nextProxy.toString());
+ }
+
+ @Test
+ void metadataComponentOverrideUsesConfiguredProxyInstanceWhenProtocolCaseDiffers() {
+ String[] proxyStrings = {
+ "http://first.example.com:8080", "HTTP://second.example.com:8080",
+ };
+ MultiProxyManager pm = new MultiProxyManager();
+ pm.configure(MultiProxyManager.ProxyRotation.LEAST_USED, proxyStrings);
+
+ SCProxy firstProxy = pm.getProxy(null).get();
+ Assertions.assertEquals("http://first.example.com:8080", firstProxy.toString());
+ Assertions.assertEquals(1, firstProxy.getUsage());
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy.host", "second.example.com");
+ metadata.setValue("http.proxy.port", "8080");
+ metadata.setValue("http.proxy.type", "HTTP");
+
+ SCProxy metadataProxy = pm.getProxy(metadata).get();
+ Assertions.assertEquals("HTTP://second.example.com:8080", metadataProxy.toString());
+ Assertions.assertEquals(1, metadataProxy.getUsage());
+
+ SCProxy nextProxy = pm.getProxy(null).get();
+ Assertions.assertEquals("http://first.example.com:8080", nextProxy.toString());
+ Assertions.assertEquals(2, nextProxy.getUsage());
+ }
+
+ @Test
+ void metadataOverrideCanUseProxyOutsideConfiguredRotation() {
+ String[] proxyStrings = {
+ "http://first.example.com:8080", "http://second.example.com:8080",
+ };
+ MultiProxyManager pm = new MultiProxyManager();
+ pm.configure(MultiProxyManager.ProxyRotation.ROUND_ROBIN, proxyStrings);
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy.host", "metadata.example.com");
+ metadata.setValue("http.proxy.port", "9443");
+ metadata.setValue("http.proxy.type", "HTTPS");
+
+ SCProxy metadataProxy = pm.getProxy(metadata).get();
+ Assertions.assertEquals("https://metadata.example.com:9443", metadataProxy.toString());
+ Assertions.assertEquals(1, metadataProxy.getUsage());
+
+ Assertions.assertEquals(
+ "http://first.example.com:8080", pm.getProxy(null).get().toString());
+ }
+
+ @Test
+ void metadataOverridePreservesReservedAuthCharacters() {
+ String[] proxyStrings = {
+ "http://first.example.com:8080", "http://second.example.com:8080",
+ };
+ MultiProxyManager pm = new MultiProxyManager();
+ pm.configure(MultiProxyManager.ProxyRotation.ROUND_ROBIN, proxyStrings);
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy.host", "metadata.example.com");
+ metadata.setValue("http.proxy.port", "9443");
+ metadata.setValue("http.proxy.user", "metadata:user");
+ metadata.setValue("http.proxy.pass", "metadata:pass@token");
+
+ SCProxy metadataProxy = pm.getProxy(metadata).get();
+ Assertions.assertEquals("metadata:user", metadataProxy.getUsername());
+ Assertions.assertEquals("metadata:pass@token", metadataProxy.getPassword());
+ Assertions.assertEquals(1, metadataProxy.getUsage());
+ }
+
+ @Test
+ void invalidMetadataProxyFailsFast() {
+ String[] proxyStrings = {
+ "http://first.example.com:8080", "http://second.example.com:8080",
+ };
+ MultiProxyManager pm = new MultiProxyManager();
+ pm.configure(MultiProxyManager.ProxyRotation.ROUND_ROBIN, proxyStrings);
+
+ assertInvalidMetadataProxyPort(pm, "0");
+ assertInvalidMetadataProxyPort(pm, "65536");
+ assertInvalidMetadataProxyPort(pm, "not-a-port");
+ assertInvalidMetadataProxyAuth(pm, "metadata-user", null);
+ assertInvalidMetadataProxyAuth(pm, null, "metadata-pass");
+ }
+
+ private void assertInvalidMetadataProxyPort(MultiProxyManager pm, String port) {
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy.host", "metadata.example.com");
+ metadata.setValue("http.proxy.port", port);
+
+ Assertions.assertThrows(IllegalArgumentException.class, () -> pm.getProxy(metadata));
+ }
+
+ private void assertInvalidMetadataProxyAuth(
+ MultiProxyManager pm, String username, String password) {
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy.host", "metadata.example.com");
+ metadata.setValue("http.proxy.port", "8080");
+ if (username != null) {
+ metadata.setValue("http.proxy.user", username);
+ }
+ if (password != null) {
+ metadata.setValue("http.proxy.pass", password);
+ }
+
+ Assertions.assertThrows(IllegalArgumentException.class, () -> pm.getProxy(metadata));
+ }
+
private SCProxy assertAndGetProxyCreation(
Supplier<Optional<SCProxy>> proxySupplier, boolean validateProxyContent) {
Optional<SCProxy> proxyOptional = proxySupplier.get();
diff --git a/core/src/test/java/org/apache/stormcrawler/proxy/SingleProxyManagerTest.java b/core/src/test/java/org/apache/stormcrawler/proxy/SingleProxyManagerTest.java
index 9635e518..239e3d31 100644
--- a/core/src/test/java/org/apache/stormcrawler/proxy/SingleProxyManagerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/proxy/SingleProxyManagerTest.java
@@ -19,6 +19,7 @@ package org.apache.stormcrawler.proxy;
import java.util.Optional;
import org.apache.storm.Config;
+import org.apache.stormcrawler.Metadata;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -44,4 +45,266 @@ class SingleProxyManagerTest {
Assertions.assertEquals("pass1", proxy.getPassword());
Assertions.assertEquals("http://user1:pass1@example.com:8080", proxy.toString());
}
+
+ @Test
+ void configuredProxyFieldsCanUseNoAuth() {
+ Config config = new Config();
+ config.put("http.proxy.host", "example.com");
+ config.put("http.proxy.type", "HTTP");
+ config.put("http.proxy.port", 8080);
+ SingleProxyManager pm = new SingleProxyManager();
+ pm.configure(config);
+
+ Optional<SCProxy> proxyOptional = pm.getProxy(null);
+ Assertions.assertTrue(proxyOptional.isPresent());
+ SCProxy proxy = proxyOptional.get();
+ Assertions.assertNull(proxy.getUsername());
+ Assertions.assertNull(proxy.getPassword());
+ Assertions.assertEquals("http://example.com:8080", proxy.toString());
+ }
+
+ @Test
+ void configuredProxyFieldsPreserveReservedAuthCharacters() {
+ Config config = new Config();
+ config.put("http.proxy.host", "example.com");
+ config.put("http.proxy.type", "HTTP");
+ config.put("http.proxy.port", 8080);
+ config.put("http.proxy.user", "user:name");
+ config.put("http.proxy.pass", "pass:word@token");
+ SingleProxyManager pm = new SingleProxyManager();
+ pm.configure(config);
+
+ Optional<SCProxy> proxyOptional = pm.getProxy(null);
+ Assertions.assertTrue(proxyOptional.isPresent());
+ SCProxy proxy = proxyOptional.get();
+ Assertions.assertEquals("user:name", proxy.getUsername());
+ Assertions.assertEquals("pass:word@token", proxy.getPassword());
+ }
+
+ @Test
+ void configuredProxyPortMustBeInRange() {
+ assertInvalidConfiguredProxyPort(0);
+ assertInvalidConfiguredProxyPort(65536);
+ }
+
+ @Test
+ void metadataConnectionStringOverridesConfiguredProxy() {
+ SingleProxyManager pm = new SingleProxyManager();
+ pm.configure(configuredSingleProxy());
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy", "https://metadata.example.com:9443");
+
+ Optional<SCProxy> proxyOptional = pm.getProxy(metadata);
+ Assertions.assertTrue(proxyOptional.isPresent());
+ Assertions.assertEquals(
+ "https://metadata.example.com:9443", proxyOptional.get().toString());
+ }
+
+ @Test
+ void metadataConnectionStringOverridesComponentFields() {
+ SingleProxyManager pm = new SingleProxyManager();
+ pm.configure(configuredSingleProxy());
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy", "https://full.example.com:9443");
+ metadata.setValue("http.proxy.host", "component.example.com");
+ metadata.setValue("http.proxy.port", "8081");
+
+ Optional<SCProxy> proxyOptional = pm.getProxy(metadata);
+ Assertions.assertTrue(proxyOptional.isPresent());
+ Assertions.assertEquals("https://full.example.com:9443", proxyOptional.get().toString());
+ }
+
+ @Test
+ void metadataProxyFieldsOverrideConfiguredProxy() {
+ SingleProxyManager pm = new SingleProxyManager();
+ pm.configure(configuredSingleProxy());
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy.host", "metadata.example.com");
+ metadata.setValue("http.proxy.port", "9443");
+ metadata.setValue("http.proxy.type", "HTTPS");
+ metadata.setValue("http.proxy.user", "metadata-user");
+ metadata.setValue("http.proxy.pass", "metadata-pass");
+
+ Optional<SCProxy> proxyOptional = pm.getProxy(metadata);
+ Assertions.assertTrue(proxyOptional.isPresent());
+ Assertions.assertEquals(
+ "https://metadata-user:metadata-pass@metadata.example.com:9443",
+ proxyOptional.get().toString());
+ }
+
+ @Test
+ void metadataProxyFieldsPreserveReservedAuthCharacters() {
+ SingleProxyManager pm = new SingleProxyManager();
+ pm.configure(configuredSingleProxy());
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy.host", "metadata.example.com");
+ metadata.setValue("http.proxy.port", "9443");
+ metadata.setValue("http.proxy.user", "metadata:user");
+ metadata.setValue("http.proxy.pass", "metadata:pass@token");
+
+ Optional<SCProxy> proxyOptional = pm.getProxy(metadata);
+ Assertions.assertTrue(proxyOptional.isPresent());
+ SCProxy proxy = proxyOptional.get();
+ Assertions.assertEquals("metadata:user", proxy.getUsername());
+ Assertions.assertEquals("metadata:pass@token", proxy.getPassword());
+ }
+
+ @Test
+ void metadataSkipDisablesConfiguredProxy() {
+ SingleProxyManager pm = new SingleProxyManager();
+ pm.configure(configuredSingleProxy());
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy.skip", "true");
+
+ Assertions.assertTrue(pm.getProxy(metadata).isEmpty());
+ }
+
+ @Test
+ void metadataSkipWinsOverMetadataOverride() {
+ SingleProxyManager pm = new SingleProxyManager();
+ pm.configure(configuredSingleProxy());
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy.skip", "true");
+ metadata.setValue("http.proxy", "http://metadata.example.com:8080");
+
+ Assertions.assertTrue(pm.getProxy(metadata).isEmpty());
+ }
+
+ @Test
+ void metadataSkipFalseUsesConfiguredProxy() {
+ SingleProxyManager pm = new SingleProxyManager();
+ pm.configure(configuredSingleProxy());
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy.skip", "false");
+
+ Optional<SCProxy> proxyOptional = pm.getProxy(metadata);
+ Assertions.assertTrue(proxyOptional.isPresent());
+ Assertions.assertEquals(
+ "http://user1:pass1@example.com:8080", proxyOptional.get().toString());
+ }
+
+ @Test
+ void invalidMetadataSkipFailsFast() {
+ SingleProxyManager pm = new SingleProxyManager();
+ pm.configure(configuredSingleProxy());
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy.skip", "not-a-boolean");
+
+ Assertions.assertThrows(IllegalArgumentException.class, () -> pm.getProxy(metadata));
+ }
+
+ @Test
+ void explicitManagerWithoutDefaultProxySupportsMetadataOnlyUrls() {
+ SingleProxyManager pm = new SingleProxyManager();
+ pm.configure(new Config());
+
+ Assertions.assertTrue(pm.getProxy(new Metadata()).isEmpty());
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy", "http://metadata-only.example.com:8080");
+
+ Optional<SCProxy> proxyOptional = pm.getProxy(metadata);
+ Assertions.assertTrue(proxyOptional.isPresent());
+ Assertions.assertEquals(
+ "http://metadata-only.example.com:8080", proxyOptional.get().toString());
+ }
+
+ @Test
+ void invalidMetadataProxyFailsFast() {
+ SingleProxyManager pm = new SingleProxyManager();
+ pm.configure(configuredSingleProxy());
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy", "metadata.example.com:8080");
+
+ Assertions.assertThrows(IllegalArgumentException.class, () -> pm.getProxy(metadata));
+ }
+
+ @Test
+ void invalidMetadataProxyDoesNotLeakCredentialsInMessage() {
+ SingleProxyManager pm = new SingleProxyManager();
+ pm.configure(configuredSingleProxy());
+
+ Metadata metadata = new Metadata();
+ metadata.setValue(
+ "http.proxy",
+ "http://metadata-user:metadata-secret@metadata.example.com:not-a-port");
+
+ IllegalArgumentException exception =
+ Assertions.assertThrows(
+ IllegalArgumentException.class, () -> pm.getProxy(metadata));
+ Assertions.assertEquals(
+ "metadata key `http.proxy` must be a valid proxy connection string",
+ exception.getMessage());
+ Assertions.assertFalse(exception.getMessage().contains("metadata-user"));
+ Assertions.assertFalse(exception.getMessage().contains("metadata-secret"));
+ Assertions.assertNull(exception.getCause());
+ }
+
+ @Test
+ void invalidMetadataProxyPortsFailFast() {
+ assertInvalidMetadataProxyPort("0");
+ assertInvalidMetadataProxyPort("65536");
+ assertInvalidMetadataProxyPort("not-a-port");
+ }
+
+ @Test
+ void metadataProxyFieldsRequireCompleteAuth() {
+ assertInvalidMetadataProxyAuth("metadata-user", null);
+ assertInvalidMetadataProxyAuth(null, "metadata-pass");
+ }
+
+ private void assertInvalidConfiguredProxyPort(int port) {
+ Config config = configuredSingleProxy();
+ config.put("http.proxy.port", port);
+
+ SingleProxyManager pm = new SingleProxyManager();
+ Assertions.assertThrows(IllegalArgumentException.class, () -> pm.configure(config));
+ }
+
+ private void assertInvalidMetadataProxyPort(String port) {
+ SingleProxyManager pm = new SingleProxyManager();
+ pm.configure(configuredSingleProxy());
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy.host", "metadata.example.com");
+ metadata.setValue("http.proxy.port", port);
+
+ Assertions.assertThrows(IllegalArgumentException.class, () -> pm.getProxy(metadata));
+ }
+
+ private void assertInvalidMetadataProxyAuth(String username, String password) {
+ SingleProxyManager pm = new SingleProxyManager();
+ pm.configure(configuredSingleProxy());
+
+ Metadata metadata = new Metadata();
+ metadata.setValue("http.proxy.host", "metadata.example.com");
+ metadata.setValue("http.proxy.port", "8080");
+ if (username != null) {
+ metadata.setValue("http.proxy.user", username);
+ }
+ if (password != null) {
+ metadata.setValue("http.proxy.pass", password);
+ }
+
+ Assertions.assertThrows(IllegalArgumentException.class, () -> pm.getProxy(metadata));
+ }
+
+ private Config configuredSingleProxy() {
+ Config config = new Config();
+ config.put("http.proxy.host", "example.com");
+ config.put("http.proxy.type", "HTTP");
+ config.put("http.proxy.port", 8080);
+ config.put("http.proxy.user", "user1");
+ config.put("http.proxy.pass", "pass1");
+ return config;
+ }
}
diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc
index 2d8a1009..69ceb545 100644
--- a/docs/src/main/asciidoc/configuration.adoc
+++ b/docs/src/main/asciidoc/configuration.adoc
@@ -55,7 +55,7 @@ This is what the configuration `http.robots.agents` allows you to do. It is a co
StormCrawler's proxy system is built on top of the link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/proxy/SCProxy.java[SCProxy] class and the link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/proxy/ProxyManager.java[ProxyManager] interface. Every proxy used in the system is formatted as a **SCProxy**. The **ProxyManager** implementations handle the management and delegation of their internal pr [...]
-The **ProxyManager** interface can be implemented in a custom class to create custom logic for proxy management and load balancing. The default **ProxyManager** implementation is **SingleProxyManager**. This ensures backwards compatibility for prior StormCrawler releases. To use **MultiProxyManager** or custom implementations, pass the class path and name via the config parameter `http.proxy.manager`:
+The **ProxyManager** interface can be implemented in a custom class to create custom logic for proxy management and load balancing. When `http.proxy.manager` is omitted, the core HTTP protocol implementations create **SingleProxyManager** only if `http.proxy.host` is configured. To use **SingleProxyManager** with only a full `http.proxy` connection string, or to use **MultiProxyManager** or custom implementations, pass the class path and name via the config parameter `http.proxy.manager`:
----
http.proxy.manager: "org.apache.stormcrawler.proxy.MultiProxyManager"
@@ -67,6 +67,7 @@ StormCrawler implements two **ProxyManager** classes by default:
Manages a single proxy passed by the backwards compatible proxy fields in the configuration:
----
+ http.proxy
http.proxy.host
http.proxy.port
http.proxy.type
@@ -92,6 +93,16 @@ Selects the proxy with the least amount of usage. This is performed lazily for s
The **SCProxy** class contains all of the information associated with proxy connection. In addition, it tracks the total usage of the proxy and optionally tracks the location of the proxy IP. Usage information is used for the **LEAST_USED** load balancing scheme. The location information is currently unused but left to enable custom implementations the ability to select proxies by location.
+The built-in proxy managers also support per-URL proxy selection from metadata. Metadata is checked before configured defaults or rotation. Set `http.proxy.skip` to `true` in a URL's metadata to fetch that URL without a proxy; this takes precedence over both metadata overrides and configured proxies. To override the proxy, either set `http.proxy` to a full proxy connection string such as `http://user:pass@proxy.example.com:8080`, or use the same component keys as the configuration: `http [...]
+
+When credentials contain URL-reserved separator characters such as `:` or `@`, prefer the component keys (`http.proxy.user` and `http.proxy.pass`) so the values are passed as fields. Component key values preserve those separator characters but trim leading and trailing whitespace. Full `http.proxy` connection strings and `http.proxy.file` entries are parsed as **SCProxy** connection strings.
+
+Metadata component authentication is fail-fast: blank `http.proxy.user` and `http.proxy.pass` values are treated as unset, but if either non-blank value is set, both must be non-blank. Configured **SingleProxyManager** component authentication is more permissive for backwards compatibility and uses unauthenticated proxying unless both configured values are non-empty.
+
+With **SingleProxyManager**, explicitly configuring `http.proxy.manager` without a default proxy is valid. URLs with proxy metadata use the metadata proxy, and URLs without proxy metadata return no proxy.
+
+With **MultiProxyManager**, a metadata proxy matching one of the configured proxies reuses the configured `SCProxy` instance so usage accounting remains meaningful. Metadata proxies outside the configured rotation are allowed and are used only for that request.
+
=== Metadata
==== Registering Metadata for Kryo Serialization
@@ -192,10 +203,15 @@ is defined.
| http.content.limit | -1 | Maximum HTTP response body size (bytes). Default: no limit.
| http.protocol.implementation | org.apache.stormcrawler.protocol.httpclient.HttpProtocol | HTTP Protocol
implementation.
-| http.proxy.host | - | HTTP proxy host.
-| http.proxy.pass | - | Proxy password.
-| http.proxy.port | 8080 | Proxy port.
-| http.proxy.user | - | Proxy username.
+| http.proxy | - | Full **SCProxy** connection string read by SingleProxyManager when that manager is selected, e.g. `http://user:pass@proxy.example.com:8080`.
+| http.proxy.file | - | Proxy connection-string file read by MultiProxyManager when that manager is selected.
+| http.proxy.host | - | SingleProxyManager component proxy host.
+| http.proxy.manager | - | ProxyManager implementation class. If omitted, the core HTTP protocols create SingleProxyManager only when `http.proxy.host` is set.
+| http.proxy.pass | - | SingleProxyManager component proxy password, used with `http.proxy.user`.
+| http.proxy.port | 8080 | SingleProxyManager component proxy port. Must be from 1 through 65535.
+| http.proxy.rotation | ROUND_ROBIN | MultiProxyManager proxy rotation scheme when that manager is selected.
+| http.proxy.type | HTTP | SingleProxyManager component proxy type.
+| http.proxy.user | - | SingleProxyManager component proxy username, used with `http.proxy.pass`.
| http.retry.on.connection.failure | true | (OkHttp only) Retry fetching on connection failure. See link:https://square.github.io/okhttp/5.x/okhttp/okhttp3/-ok-http-client/-builder/retry-on-connection-failure.html[OkHttp docs].
| http.robots.403.allow | true | Allow crawling when robots.txt returns HTTP 403.
| http.robots.5xx.allow | false | Allow crawling when robots.txt returns a server error (5xx).
diff --git a/docs/src/main/asciidoc/internals.adoc b/docs/src/main/asciidoc/internals.adoc
index 4896e18d..65746568 100644
--- a/docs/src/main/asciidoc/internals.adoc
+++ b/docs/src/main/asciidoc/internals.adoc
@@ -475,6 +475,12 @@ The `metadata` argument to link:https://github.com/apache/stormcrawler/blob/main
* `protocol.set-headers`: If this key is present in metadata, the protocol adds the specified headers to the request. See link:https://github.com/apache/stormcrawler/pull/993[#993]
+* `http.proxy.skip`: If this key is set to `true` in `metadata`, the built-in proxy managers return no proxy for the request, even when a default proxy, proxy rotation, or metadata proxy override is configured. If present, the value must be `true` or `false`.
+
+* `http.proxy`: If this key is present in `metadata`, the built-in proxy managers use it as a full proxy connection string for the request, e.g. `http://user:pass@proxy.example.com:8080`. It takes precedence over component proxy metadata keys. Invalid values throw `IllegalArgumentException`.
+
+* `http.proxy.host`, `http.proxy.port`, `http.proxy.type`, `http.proxy.user`, `http.proxy.pass`: If these keys are present in `metadata`, the built-in proxy managers build a per-request proxy from the component values. These metadata keys use the same names and defaults as the proxy configuration keys, except blank `http.proxy.user` and `http.proxy.pass` values are treated as unset, and non-blank authentication values must be supplied together. Component metadata values are trimmed. The [...]
+
Example:
[source,json]
@@ -506,5 +512,3 @@ metadata.persist:
metadata.transfer:
- set-cookie
----
-
-