(tika) 15/24: TIKA-4809: Make http-fetcher TLS verification reachable, and on by default

[ta...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

tballison pushed a commit to branch TIKA-4809-stage-9
in repository https://gitbox.apache.org/repos/asf/tika.git

commit 6548ea8a8e6da0c4412f495fc4acce0fe6d29afa
Author: tallison <ta...@apache.org>
AuthorDate: Tue Aug 11 06:51:45 2026 -0400

    TIKA-4809: Make http-fetcher TLS verification reachable, and on by default
---
 docs/modules/ROOT/pages/pipes/plugins/http.adoc    | 50 +++++++++++++++-------
 .../tika/pipes/fetcher/http/HttpFetcher.java       | 23 +++++++---
 .../fetcher/http/config/HttpFetcherConfig.java     | 18 ++++++++
 3 files changed, 69 insertions(+), 22 deletions(-)

diff --git a/docs/modules/ROOT/pages/pipes/plugins/http.adoc b/docs/modules/ROOT/pages/pipes/plugins/http.adoc
index 9acab6fbce..74960a32eb 100644
--- a/docs/modules/ROOT/pages/pipes/plugins/http.adoc
+++ b/docs/modules/ROOT/pages/pipes/plugins/http.adoc
@@ -92,7 +92,11 @@ include::example$pipes-http-fetcher.json[]
 
 |`maxRedirects`
 |`0`
-|Maximum number of redirects to follow. `0` means follow none. Not applied to range requests — see <<security-notes>>.
+|Maximum number of redirects to follow. `0` means follow none.
+
+|`verifySsl`
+|`true`
+|Verify server certificates and hostnames. Set `false` to accept any certificate — see <<security-notes>>.
 
 |`maxSpoolSize`
 |`-1`
@@ -139,21 +143,35 @@ endpoint that can reach it (`/pipes`, `/async`).
   Schemes other than `http`/`https` fail only because no other scheme is registered in
   the connection manager — that is a side effect of the transport setup, not a check.
 
-* **TLS certificates are not verified, and this is not configurable here.** The
-  underlying client defaults to `verifySsl=false`, which installs an accept-everything
-  trust strategy and `NoopHostnameVerifier`. `HttpFetcherConfig` exposes no `verifySsl`
-  setting, so an http-fetcher config cannot turn verification on. Do not use this fetcher
-  to retrieve anything whose authenticity matters over an untrusted network.
-
-* **`maxRedirects` does not apply to range requests.** The main `fetch` builds a
-  `RequestConfig` from `maxRedirects`; the `startRange`/`endRange` overload sets no
-  request config at all and therefore uses the client's defaults (redirects enabled).
-  A `maxRedirects: 0` setting does not stop redirects on a range fetch.
-
-* **The redirect host allowlist is currently inert.** `CustomRedirectStrategy` will
-  refuse a redirect to a host outside `allowedHostsForRedirect`, but that set is never
-  populated from any configuration path, and the check is skipped when the set is empty.
-  Do not rely on it to contain redirects.
+* **TLS certificates are verified by default (`verifySsl: true`).** Earlier 4.0.0
+  prereleases could not verify at all: the underlying client defaulted to `false` and
+  `HttpFetcherConfig` exposed no setting, so no configuration could turn verification on.
++
+To opt out -- self-signed internal certificates are the usual reason -- set `verifySsl`
+to `false` on the fetcher:
++
+[source,json]
+----
+{
+  "fetchers": {
+    "my-http-fetcher": {
+      "plugin": "http-fetcher",
+      "config": { "verifySsl": false }
+    }
+  }
+}
+----
++
+That installs an accept-everything trust strategy and `NoopHostnameVerifier`: any
+certificate from any host is accepted, so anything on the network path can read and
+alter what is fetched. Prefer adding your CA to the JVM truststore over turning this off.
+
+* **The redirect host allowlist is inert, and would not be trustworthy if it were not.**
+  `CustomRedirectStrategy` refuses a redirect to a host outside `allowedHostsForRedirect`,
+  but that set is never populated from any configuration path, and the check is skipped
+  when the set is empty. It also allows the redirect outright when the `Location` header
+  fails to parse as a URI, so the allowlist would have a bypass even once populated. Do
+  not rely on it to contain redirects.
 
 [#notes]
 == Notes
diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java b/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
index dad55c9e55..bbaa22aec7 100644
--- a/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
+++ b/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java
@@ -162,15 +162,18 @@ public class HttpFetcher extends AbstractTikaExtension implements Fetcher, Range
     public TikaInputStream fetch(String fetchKey, Metadata metadata, ParseContext parseContext) throws IOException, TikaException {
         HttpFetcherConfig additionalHttpFetcherConfig = getAdditionalHttpFetcherConfig(parseContext);
         HttpGet get = new HttpGet(fetchKey);
-        RequestConfig requestConfig = RequestConfig
+        get.setConfig(buildRequestConfig());
+        setHttpRequestHeaders(metadata, get);
+        putAdditionalHeadersOnRequest(additionalHttpFetcherConfig, get);
+        return execute(get, metadata, httpClient, true);
+    }
+
+    private RequestConfig buildRequestConfig() {
+        return RequestConfig
                 .custom()
                 .setMaxRedirects(httpFetcherConfig.getMaxRedirects())
                 .setRedirectsEnabled(httpFetcherConfig.getMaxRedirects() > 0)
                 .build();
-        get.setConfig(requestConfig);
-        setHttpRequestHeaders(metadata, get);
-        putAdditionalHeadersOnRequest(additionalHttpFetcherConfig, get);
-        return execute(get, metadata, httpClient, true);
     }
 
     private void setHttpRequestHeaders(Metadata metadata, HttpGet get) {
@@ -218,6 +221,11 @@ public class HttpFetcher extends AbstractTikaExtension implements Fetcher, Range
                              ParseContext parseContext) throws IOException, TikaException {
         HttpFetcherConfig additionalHttpFetcherConfig = getAdditionalHttpFetcherConfig(parseContext);
         HttpGet get = new HttpGet(fetchKey);
+        // Same RequestConfig and headers as the whole-document fetch above. Without the
+        // config this used client defaults, which leave redirects ENABLED -- so
+        // maxRedirects: 0 did not stop redirects on a range fetch.
+        get.setConfig(buildRequestConfig());
+        setHttpRequestHeaders(metadata, get);
         putAdditionalHeadersOnRequest(additionalHttpFetcherConfig, get);
 
         get.setHeader("Range", "bytes=" + startRange + "-" + endRange);
@@ -466,7 +474,7 @@ public class HttpFetcher extends AbstractTikaExtension implements Fetcher, Range
             httpClientFactory.setRequestTimeoutMillis(httpFetcherConfig.getRequestTimeoutMillis());
         }
         if (httpFetcherConfig.getConnectTimeoutMillis() != null) {
-            httpClientFactory.setSocketTimeoutMillis(httpFetcherConfig.getConnectTimeoutMillis());
+            httpClientFactory.setConnectTimeoutMillis(httpFetcherConfig.getConnectTimeoutMillis());
         }
         if (httpFetcherConfig.getMaxConnections() != null) {
             httpClientFactory.setMaxConnections(httpFetcherConfig.getMaxConnections());
@@ -474,6 +482,9 @@ public class HttpFetcher extends AbstractTikaExtension implements Fetcher, Range
         if (httpFetcherConfig.getMaxConnectionsPerRoute() != null) {
             httpClientFactory.setMaxConnectionsPerRoute(httpFetcherConfig.getMaxConnectionsPerRoute());
         }
+        // Unreachable before: the factory had the setter, but nothing carried a config value
+        // to it, so TLS verification could not be turned on from any config.
+        httpClientFactory.setVerifySsl(httpFetcherConfig.isVerifySsl());
         if (!StringUtils.isBlank(httpFetcherConfig.getAuthScheme())) {
             httpClientFactory.setUserName(httpFetcherConfig.getUserName());
             httpClientFactory.setPassword(httpFetcherConfig.getPassword());
diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpFetcherConfig.java b/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpFetcherConfig.java
index dc4335f6fb..aba8a4c420 100644
--- a/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpFetcherConfig.java
+++ b/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpFetcherConfig.java
@@ -51,6 +51,15 @@ public class HttpFetcherConfig {
     private Integer socketTimeoutMillis = 120000;
     private Long maxSpoolSize = -1L;
     private Integer maxRedirects = 0;
+    /**
+     * Verify server certificates and hostnames. Defaults to true, matching the Solr
+     * plugins; previously there was no field at all, so the factory's setting could not
+     * be reached from any config and verification could not be turned on.
+     * <p>
+     * Set to false to accept any certificate from any host -- needed for self-signed
+     * internal certs, and the only supported way to opt out.
+     */
+    private boolean verifySsl = true;
     private List<String> httpHeaders = new ArrayList<>();
     private HttpHeaders httpRequestHeaders = new HttpHeaders();
     private Long overallTimeoutMillis = 120000L;
@@ -269,4 +278,13 @@ public class HttpFetcherConfig {
         this.jwtPrivateKeyBase64 = jwtPrivateKeyBase64;
         return this;
     }
+    public boolean isVerifySsl() {
+        return verifySsl;
+    }
+
+    public HttpFetcherConfig setVerifySsl(boolean verifySsl) {
+        this.verifySsl = verifySsl;
+        return this;
+    }
+
 }