You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by jn...@apache.org on 2026/07/23 14:01:47 UTC
(stormcrawler) 01/01: Mark HTTPClient as deprecated
This is an automated email from the ASF dual-hosted git repository.
jnioche pushed a commit to branch 2013
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
commit b14a57412f41e0912ccf54b7f768606cd1666582
Author: Julien Nioche <ju...@digitalpebble.com>
AuthorDate: Thu Jul 23 15:01:29 2026 +0100
Mark HTTPClient as deprecated
Signed-off-by: Julien Nioche <ju...@digitalpebble.com>
---
.../org/apache/stormcrawler/protocol/DelegatorProtocol.java | 4 ++--
.../org/apache/stormcrawler/protocol/ProtocolFactory.java | 4 ++--
.../stormcrawler/protocol/httpclient/HttpProtocol.java | 9 ++++++++-
.../org/apache/stormcrawler/proxy/MultiProxyManager.java | 3 +--
core/src/main/resources/crawler-default.yaml | 7 +++++--
.../httpclient/HttpClientProtocolProxyManagerTest.java | 1 +
docs/src/main/asciidoc/configuration.adoc | 12 ++++++++----
docs/src/main/asciidoc/internals.adoc | 6 +++---
8 files changed, 30 insertions(+), 16 deletions(-)
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java
index 087bc70b..b19760b9 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java
@@ -38,7 +38,7 @@ import org.slf4j.LoggerFactory;
*
* <pre>
* protocol.delegator.config:
- * - className: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol"
+ * - className: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
* filters:
* domain: "example.com"
* depth: "3"
@@ -50,7 +50,7 @@ import org.slf4j.LoggerFactory;
* regex:
* - \.pdf
* - \.doc
- * - className: "org.apache.stormcrawler.protocol.selenium.SeleniumProtocol"
+ * - className: "org.apache.stormcrawler.protocol.playwright.HttpProtocol"
* </pre>
*
* Typically, the last one in the list must not have filters as it is used as a default value. The
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java
index 5b870fbd..10619f8b 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java
@@ -72,10 +72,10 @@ public class ProtocolFactory {
// set the default values
if (protocol.equalsIgnoreCase("http")) {
protocolimplementation =
- "org.apache.stormcrawler.protocol.httpclient.HttpProtocol";
+ "org.apache.stormcrawler.protocol.okhttp.HttpProtocol";
} else if (protocol.equalsIgnoreCase("https")) {
protocolimplementation =
- "org.apache.stormcrawler.protocol.httpclient.HttpProtocol";
+ "org.apache.stormcrawler.protocol.okhttp.HttpProtocol";
} else {
throw new RuntimeException(paramName + "should not have an empty value");
}
diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java
index 0c67b184..9ea6a332 100644
--- a/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java
+++ b/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java
@@ -68,7 +68,14 @@ import org.apache.stormcrawler.util.ConfUtils;
import org.apache.stormcrawler.util.CookieConverter;
import org.slf4j.LoggerFactory;
-/** Uses Apache httpclient to handle http and https. */
+/**
+ * Uses Apache httpclient to handle http and https.
+ *
+ * @deprecated The Apache HttpClient protocol implementation is deprecated and will be removed in the
+ * next major release of StormCrawler. Use the OkHttp implementation ({@link
+ * org.apache.stormcrawler.protocol.okhttp.HttpProtocol}) instead.
+ */
+@Deprecated
public class HttpProtocol extends AbstractHttpProtocol
implements ResponseHandler<ProtocolResponse> {
diff --git a/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java b/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java
index 68a600d4..b65392c3 100644
--- a/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java
+++ b/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java
@@ -30,7 +30,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.storm.Config;
import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.protocol.httpclient.HttpProtocol;
import org.apache.stormcrawler.util.ConfUtils;
import org.slf4j.LoggerFactory;
@@ -48,7 +47,7 @@ public class MultiProxyManager implements ProxyManager {
private final AtomicInteger lastAccessedIndex = new AtomicInteger(0);
private Map<SCProxy, SCProxy> proxyLookupMap;
- private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(HttpProtocol.class);
+ private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MultiProxyManager.class);
/** Default constructor for setting up the proxy manager. */
private void init(ProxyRotation rotation) {
diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml
index 9961320b..f5c55104 100644
--- a/core/src/main/resources/crawler-default.yaml
+++ b/core/src/main/resources/crawler-default.yaml
@@ -194,8 +194,11 @@ config:
robots.error.cache.spec: "maximumSize=10000,expireAfterWrite=1h"
protocols: "http,https,file"
- http.protocol.implementation: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol"
- https.protocol.implementation: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol"
+ # NOTE: the Apache HttpClient protocol implementation
+ # (org.apache.stormcrawler.protocol.httpclient.HttpProtocol) is deprecated and will be removed in
+ # the next major release of StormCrawler. OkHttp is now the default.
+ http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
+ https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
file.protocol.implementation: "org.apache.stormcrawler.protocol.file.FileProtocol"
# number of instances for each protocol implementation
diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java
index d67de936..d350e8f8 100644
--- a/core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java
@@ -26,6 +26,7 @@ import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.proxy.SingleProxyManager;
import org.junit.jupiter.api.Test;
+@SuppressWarnings("deprecation")
class HttpClientProtocolProxyManagerTest {
@Test
diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc
index fd9e52b6..1b3075f5 100644
--- a/docs/src/main/asciidoc/configuration.adoc
+++ b/docs/src/main/asciidoc/configuration.adoc
@@ -201,8 +201,10 @@ is defined.
| http.basicauth.password | - | Password for http.basicauth.user.
| http.basicauth.user | - | Username for Basic Authentication.
| http.content.limit | -1 | Maximum HTTP response body size (bytes). Default: no limit.
-| http.protocol.implementation | org.apache.stormcrawler.protocol.httpclient.HttpProtocol | HTTP Protocol
-implementation.
+| http.protocol.implementation | org.apache.stormcrawler.protocol.okhttp.HttpProtocol | HTTP Protocol
+implementation. Note: the Apache HttpClient implementation
+(`org.apache.stormcrawler.protocol.httpclient.HttpProtocol`) is *deprecated* and will be removed in
+the next major release.
| http.proxy | - | Full **SCProxy** connection string read by SingleProxyManager when that manager is selected, e.g. `http://user:pass@proxy.example.com:8080`.
| http.proxy.file | - | Proxy connection-string file read by MultiProxyManager when that manager is selected.
| http.proxy.host | - | SingleProxyManager component proxy host.
@@ -225,8 +227,10 @@ implementation.
| http.store.headers | false | Whether to store response headers.
| http.timeout | 10000 | Connection timeout (ms).
| http.use.cookies | false | Use cookies in subsequent requests.
-| https.protocol.implementation | org.apache.stormcrawler.protocol.httpclient.HttpProtocol | HTTPS Protocol
-implementation.
+| https.protocol.implementation | org.apache.stormcrawler.protocol.okhttp.HttpProtocol | HTTPS Protocol
+implementation. Note: the Apache HttpClient implementation
+(`org.apache.stormcrawler.protocol.httpclient.HttpProtocol`) is *deprecated* and will be removed in
+the next major release.
| partition.url.mode | byHost | Defines how URLs are partitioned: byHost, byDomain, or byIP.
| protocols | http,https,file | Supported protocols.
| redirections.allowed | true | If true, emit redirect target URLs as "outlinks" to the status stream. If false, do not follow redirects. See also `http.allow.redirects`.
diff --git a/docs/src/main/asciidoc/internals.adoc b/docs/src/main/asciidoc/internals.adoc
index 7dc32d5c..0b008a32 100644
--- a/docs/src/main/asciidoc/internals.adoc
+++ b/docs/src/main/asciidoc/internals.adoc
@@ -395,13 +395,13 @@ The following network protocols are implemented in StormCrawler:
* link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/file/FileProtocol.java[FileProtocol]
===== DelegatorProtocol
-The link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java[DelegatorProtocol] enables selection from a collection of sub-protocols using filters based on metadata and URL patterns. The last protocol in the list acts as the default. This is useful when certain URLs need to be fetched with a browser-based protocol (e.g., Selenium) while the majority use a standard HTTP client.
+The link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java[DelegatorProtocol] enables selection from a collection of sub-protocols using filters based on metadata and URL patterns. The last protocol in the list acts as the default. This is useful when certain URLs need to be fetched with a browser-based protocol (e.g., Playwright) while the majority use a standard HTTP client.
[source,yaml]
----
http.protocol.implementation: "org.apache.stormcrawler.protocol.DelegatorProtocol"
protocol.delegator.config:
- - className: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol"
+ - className: "org.apache.stormcrawler.protocol.playwright.HttpProtocol"
filters:
domain: "example.com"
- className: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
@@ -421,7 +421,7 @@ http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProto
https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
----
-* link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java[HttpClient]
+* link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java[HttpClient] (*deprecated* — will be removed in the next major release; use OKHttp instead)
* link:https://github.com/apache/stormcrawler/blob/main/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/SeleniumProtocol.java[Selenium]
* link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java[OKHttp]