You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by dp...@apache.org on 2026/10/16 15:37:06 UTC

(stormcrawler) branch opensearch-3.x created (now baef68fa)

This is an automated email from the ASF dual-hosted git repository.

dpol1 pushed a change to branch opensearch-3.x
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git


      at baef68fa keep the connection pool limits when TLS validation is disabled

This branch includes the following new commits:

     new 8d5818e8 accept null metadata values in the opensearch-java bolts
     new 1ba7227a replace the legacy opensearch sources with the opensearch-java ones
     new a1e375e8 build external/opensearch with the opensearch-java client
     new dade9a7f drop the sniff and responseBufferSize settings
     new 7cf5c549 merge the opensearch-java README into external/opensearch
     new 9ac483d5 port the legacy tests for credentials and WaitAckCache shutdown
     new baef68fa keep the connection pool limits when TLS validation is disabled

The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(stormcrawler) 06/07: port the legacy tests for credentials and WaitAckCache shutdown

Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

dpol1 pushed a commit to branch opensearch-3.x
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git

commit 9ac483d564842678050e3d8fe9b5c6de11e5cee6
Author: Davide Polato <dp...@apache.org>
AuthorDate: Fri Oct 16 16:36:49 2026 +0200

    port the legacy tests for credentials and WaitAckCache shutdown
    
    The legacy module tested that a running client sends the credentials
    to a configured address and not to another one, that credentials need
    both a user and a password, that host and scheme are matched without
    regard to case, and that WaitAckCache.shutdown() fails the tuples
    still waiting. The opensearch-java tests only checked how the
    credential scopes are built.
    
    The tests are adapted to HttpClient 5 and the opensearch-java
    transport. Two legacy tests are not carried over: the sniffer scheme
    test, since there is no sniffer, and the test for a credentials lookup
    without an origin, since the transport's preemptive Basic scheme looks
    the credentials up with the host of the request.
    
    Signed-off-by: Davide Polato <dp...@apache.org>
---
 .../OpenSearchConnectionCredentialsTest.java       | 118 +++++++++++++++++++++
 .../stormcrawler/opensearch/WaitAckCacheTest.java  |  16 +++
 2 files changed, 134 insertions(+)

diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java
index bb484e84..ba8e176e 100644
--- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java
@@ -23,12 +23,30 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.UncheckedIOException;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.function.IntFunction;
 import org.apache.hc.client5.http.auth.AuthScope;
 import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
 import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
 import org.apache.hc.core5.http.HttpHost;
 import org.junit.jupiter.api.Test;
+import org.opensearch.client.opensearch.OpenSearchClient;
+import org.opensearch.client.opensearch.generic.Requests;
+import org.opensearch.client.opensearch.generic.Response;
+import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
 
 class OpenSearchConnectionCredentialsTest {
 
@@ -98,6 +116,106 @@ class OpenSearchConnectionCredentialsTest {
         return new AuthScope(new HttpHost(scheme, host, port), null, "Basic");
     }
 
+    @Test
+    void credentialsAreGivenToTheConfiguredAddresses() {
+        final BasicCredentialsProvider provider =
+                providerFor(
+                        List.of(
+                                new HttpHost("https", "opensearch1.example.org", 9200),
+                                new HttpHost("https", "OpenSearch2.example.org", 9201)));
+        assertNotNull(
+                provider.getCredentials(request("https", "opensearch1.example.org", 9200), null));
+        assertNotNull(
+                provider.getCredentials(request("HTTPS", "opensearch2.example.org", 9201), null));
+    }
+
+    /**
+     * Sends a request with the client built for the port of a local server and returns the
+     * Authorization header the server received, if any.
+     */
+    private static String authorizationSentTo(IntFunction<OpenSearchClient> clientForPort)
+            throws Exception {
+        try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getLoopbackAddress())) {
+            final CompletableFuture<String> authorization =
+                    CompletableFuture.supplyAsync(
+                            () -> {
+                                try (Socket socket = server.accept()) {
+                                    final BufferedReader in =
+                                            new BufferedReader(
+                                                    new InputStreamReader(
+                                                            socket.getInputStream(),
+                                                            StandardCharsets.US_ASCII));
+                                    String header = null;
+                                    for (String line = in.readLine();
+                                            line != null && !line.isEmpty();
+                                            line = in.readLine()) {
+                                        if (line.regionMatches(true, 0, "Authorization:", 0, 14)) {
+                                            header = line.substring(14).trim();
+                                        }
+                                    }
+                                    socket.getOutputStream()
+                                            .write(
+                                                    "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
+                                                            .getBytes(StandardCharsets.US_ASCII));
+                                    return header;
+                                } catch (IOException e) {
+                                    throw new UncheckedIOException(e);
+                                }
+                            });
+            final OpenSearchClient client = clientForPort.apply(server.getLocalPort());
+            try (Response response =
+                    client.generic()
+                            .execute(Requests.builder().endpoint("/").method("GET").build())) {
+                return authorization.get(10, TimeUnit.SECONDS);
+            } finally {
+                client._transport().close();
+            }
+        }
+    }
+
+    private static OpenSearchClient connectionClient(int port, String user, String password) {
+        final Map<String, Object> conf = new HashMap<>();
+        conf.put("opensearch.test.addresses", "http://127.0.0.1:" + port);
+        conf.put("opensearch.test.user", user);
+        conf.put("opensearch.test.password", password);
+        return OpenSearchConnection.getClient(conf, "test");
+    }
+
+    @Test
+    void clientSendsTheCredentialsToAConfiguredAddress() throws Exception {
+        assertEquals(
+                "Basic "
+                        + Base64.getEncoder()
+                                .encodeToString("crawler:s3cret".getBytes(StandardCharsets.UTF_8)),
+                authorizationSentTo(port -> connectionClient(port, "crawler", "s3cret")));
+    }
+
+    @Test
+    void clientSendsNoCredentialsToAnotherAddress() throws Exception {
+        assertNull(
+                authorizationSentTo(
+                        port ->
+                                new OpenSearchClient(
+                                        ApacheHttpClient5TransportBuilder.builder(
+                                                        new HttpHost("http", "127.0.0.1", port))
+                                                .setHttpClientConfigCallback(
+                                                        b ->
+                                                                b.setDefaultCredentialsProvider(
+                                                                        providerFor(
+                                                                                List.of(
+                                                                                        new HttpHost(
+                                                                                                "https",
+                                                                                                "127.0.0.1",
+                                                                                                port)))))
+                                                .build())));
+    }
+
+    @Test
+    void credentialsNeedUserAndPassword() throws Exception {
+        assertNull(authorizationSentTo(port -> connectionClient(port, "crawler", " ")));
+        assertNull(authorizationSentTo(port -> connectionClient(port, null, "s3cret")));
+    }
+
     @Test
     void plainHttpToARemoteHostIsReported() {
         final HttpHost remote = new HttpHost("http", "opensearch1.example.org", 9200);
diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java
index 3bd2dc6d..0b7954a6 100644
--- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java
@@ -333,4 +333,20 @@ class WaitAckCacheTest {
         assertEquals(1, failed.size());
         assertSame(t2, failed.get(0));
     }
+
+    @Test
+    void shutdown_failsAllRemainingTuples() {
+        Tuple t1 = mockTuple("http://example.com/1");
+        Tuple t2 = mockTuple("http://example.com/2");
+        cache.addTuple("doc1", t1);
+        cache.addTuple("doc2", t2);
+
+        cache.shutdown();
+
+        assertEquals(2, evicted.size());
+        assertTrue(evicted.contains(t1));
+        assertTrue(evicted.contains(t2));
+        assertFalse(cache.contains("doc1"));
+        assertFalse(cache.contains("doc2"));
+    }
 }


(stormcrawler) 01/07: accept null metadata values in the opensearch-java bolts

Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

dpol1 pushed a commit to branch opensearch-3.x
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git

commit 8d5818e85cef30e95743336d4f81782a96a7fd71
Author: Davide Polato <dp...@apache.org>
AuthorDate: Fri Oct 16 15:54:21 2026 +0200

    accept null metadata values in the opensearch-java bolts
    
    Metadata.setValue(key, null) stores a null element, and addValue
    keeps it next to the new value. StatusUpdaterBolt copied the values
    of every metadata key with List.of, and IndexerBolt the values of the
    keys with more than one value. List.of throws a NullPointerException
    on a null element; the bolts caught it and failed the tuple.
    
    Both bolts now copy the values with Arrays.asList, and the null is
    written as a JSON null inside the array, as the legacy module does.
    
    Signed-off-by: Davide Polato <dp...@apache.org>
---
 .../stormcrawler/opensearch/bolt/IndexerBolt.java  |   4 +-
 .../opensearch/persistence/StatusUpdaterBolt.java  |   4 +-
 .../opensearch/bolt/NullMetadataValueTest.java     | 146 +++++++++++++++++++++
 3 files changed, 150 insertions(+), 4 deletions(-)

diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
index c98a0aba..f7a44185 100644
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
+++ b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
@@ -20,8 +20,8 @@ package org.apache.stormcrawler.opensearch.bolt;
 import static org.apache.stormcrawler.Constants.StatusStreamName;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import org.apache.commons.lang3.StringUtils;
@@ -183,7 +183,7 @@ public class IndexerBolt extends AbstractIndexerBolt implements AsyncBulkProcess
                         source.put(entry.getKey(), value);
                     }
                 } else if (entry.getValue().length > 1) {
-                    source.put(entry.getKey(), List.of(entry.getValue()));
+                    source.put(entry.getKey(), Arrays.asList(entry.getValue()));
                 }
             }
 
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
index bc0c59cc..2dcd76fe 100644
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
+++ b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
@@ -18,9 +18,9 @@
 package org.apache.stormcrawler.opensearch.persistence;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
@@ -231,7 +231,7 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt
             String[] values = metadata.getValues(mdKey);
             // periods are not allowed - replace with %2E
             mdKey = mdKey.replaceAll("\\.", "%2E");
-            metadataMap.put(mdKey, List.of(values));
+            metadataMap.put(mdKey, Arrays.asList(values));
         }
 
         String partitionKey = partitioner.getPartition(url, metadata);
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java
new file mode 100644
index 00000000..17f04522
--- /dev/null
+++ b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.opensearch.bolt;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.topology.base.BaseRichBolt;
+import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestOutputCollector;
+import org.apache.stormcrawler.TestUtil;
+import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
+import org.apache.stormcrawler.opensearch.IndexCreation;
+import org.apache.stormcrawler.opensearch.OpenSearchConnection;
+import org.apache.stormcrawler.opensearch.persistence.StatusUpdaterBolt;
+import org.apache.stormcrawler.persistence.AbstractStatusUpdaterBolt;
+import org.apache.stormcrawler.persistence.DefaultScheduler;
+import org.apache.stormcrawler.persistence.Scheduler;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.util.MetadataTransfer;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.MockedStatic;
+import org.opensearch.client.opensearch.core.bulk.BulkOperation;
+
+/**
+ * Metadata values can be null. Runs without OpenSearch: the static connection factory and index
+ * creation are mocked, and the bulk operation passed to the connection is inspected.
+ */
+class NullMetadataValueTest {
+
+    private MockedStatic<OpenSearchConnection> connectionFactory;
+
+    private MockedStatic<IndexCreation> indexCreation;
+
+    private OpenSearchConnection connection;
+
+    private TestOutputCollector output;
+
+    private BaseRichBolt bolt;
+
+    @BeforeEach
+    void mockConnection() {
+        connection = mock(OpenSearchConnection.class);
+        connectionFactory = mockStatic(OpenSearchConnection.class);
+        connectionFactory
+                .when(() -> OpenSearchConnection.getConnection(any(), anyString(), any()))
+                .thenReturn(connection);
+        indexCreation = mockStatic(IndexCreation.class);
+        output = new TestOutputCollector();
+    }
+
+    @AfterEach
+    void cleanup() {
+        try {
+            // fails the tuples still waiting for a bulk response, so it runs after the assertions
+            bolt.cleanup();
+        } finally {
+            indexCreation.close();
+            connectionFactory.close();
+        }
+    }
+
+    private static Metadata metadataWithNullValue() {
+        Metadata md = new Metadata();
+        md.setValue("key", null);
+        md.addValue("key", "value");
+        return md;
+    }
+
+    private static Tuple tuple() {
+        Tuple tuple = mock(Tuple.class);
+        when(tuple.getStringByField("url")).thenReturn("https://example.com/");
+        when(tuple.getValueByField("status")).thenReturn(Status.FETCHED);
+        when(tuple.getValueByField("metadata")).thenReturn(metadataWithNullValue());
+        return tuple;
+    }
+
+    @SuppressWarnings("unchecked")
+    private Map<String, Object> sentDocument() {
+        ArgumentCaptor<BulkOperation> op = ArgumentCaptor.forClass(BulkOperation.class);
+        verify(connection).addToProcessor(op.capture());
+        return (Map<String, Object>) op.getValue().index().document();
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    void statusUpdaterBoltSendsNullValue() {
+        Map<String, Object> conf = new HashMap<>();
+        conf.put(Scheduler.schedulerClassParamName, DefaultScheduler.class.getName());
+        conf.put(
+                AbstractStatusUpdaterBolt.cacheConfigParamName,
+                "maximumSize=10000,expireAfterAccess=1h");
+        conf.put(MetadataTransfer.metadataPersistParamName, "key");
+        bolt = new StatusUpdaterBolt();
+        bolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+
+        bolt.execute(tuple());
+
+        assertTrue(output.getFailedTuples().isEmpty());
+        Map<String, Object> metadata = (Map<String, Object>) sentDocument().get("metadata");
+        assertEquals(Arrays.asList(null, "value"), metadata.get("key"));
+    }
+
+    @Test
+    void indexerBoltSendsNullValue() {
+        Map<String, Object> conf = new HashMap<>();
+        conf.put(AbstractIndexerBolt.urlFieldParamName, "url");
+        conf.put(AbstractIndexerBolt.metadata2fieldParamName, "key");
+        bolt = new IndexerBolt("content");
+        bolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+
+        bolt.execute(tuple());
+
+        assertTrue(output.getFailedTuples().isEmpty());
+        assertEquals(Arrays.asList(null, "value"), sentDocument().get("key"));
+    }
+}


(stormcrawler) 05/07: merge the opensearch-java README into external/opensearch

Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

dpol1 pushed a commit to branch opensearch-3.x
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git

commit 7cf5c549dd89088c3ffc2ff0e1e0d3f7df2de810
Author: Davide Polato <dp...@apache.org>
AuthorDate: Fri Oct 16 16:10:04 2026 +0200

    merge the opensearch-java README into external/opensearch
    
    The README of external/opensearch describes the client and
    AsyncBulkProcessor, and keeps the archetype and dashboard instructions.
    A new section lists what changes for users of earlier versions: the
    configuration keys, the behaviour differences and the types that custom
    code has to use.
    
    Signed-off-by: Davide Polato <dp...@apache.org>
---
 external/opensearch-java/README.md |  83 ---------------------------
 external/opensearch/README.md      | 114 ++++++++++++++++++++++++++++++++++---
 2 files changed, 106 insertions(+), 91 deletions(-)

diff --git a/external/opensearch-java/README.md b/external/opensearch-java/README.md
deleted file mode 100644
index d1cf8e8e..00000000
--- a/external/opensearch-java/README.md
+++ /dev/null
@@ -1,83 +0,0 @@
-stormcrawler-opensearch-java
-===========================
-
-A collection of resources for [OpenSearch](https://opensearch.org/) built on the
-[OpenSearch Java Client 3.x](https://opensearch.org/docs/latest/clients/java/) and
-Apache HttpClient 5:
-
-* [IndexerBolt](https://github.com/apache/stormcrawler/blob/master/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java) for indexing documents crawled with StormCrawler
-* [Spouts](https://github.com/apache/stormcrawler/blob/master/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java) and [StatusUpdaterBolt](https://github.com/apache/stormcrawler/blob/master/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java) for persisting URL information in recursive crawls
-* [MetricsConsumer](https://github.com/apache/stormcrawler/blob/master/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java)
-* [StatusMetricsBolt](https://github.com/apache/stormcrawler/blob/master/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java) for sending the breakdown of URLs per status as metrics and display its evolution over time.
-
-The bolts write to OpenSearch through `AsyncBulkProcessor`, which groups their
-operations into bulk requests and:
-
-* sends a bulk request when the number of buffered operations reaches
-  `opensearch.<bolt>.bulkActions`, when the buffered operations reach 5 MiB of
-  JSON (counted before compression), or when `opensearch.<bolt>.flushInterval`
-  elapses.
-* resends the items that OpenSearch rejects with status 429 (too many requests)
-  up to 8 times, after 50, 60, 80, 150, 280, 580, 1250 and 2740 ms.
-  `IndexerBolt`, `DeletionBolt` and `StatusUpdaterBolt` fail the tuple of an
-  item still rejected after the last retry.
-* sends the requests without blocking the bolt. The bolt waits only when
-  `opensearch.<bolt>.concurrentRequests` bulks are in flight, including bulks
-  waiting for a retry.
-* reports the outcome of a bulk to the bolt once, after its last attempt, with
-  the items in request order.
-
-Getting started
----------------------
-
-Add the dependency to your crawler project:
-
-```xml
-<dependency>
-    <groupId>org.apache.stormcrawler</groupId>
-    <artifactId>stormcrawler-opensearch-java</artifactId>
-    <version>${stormcrawler.version}</version>
-</dependency>
-```
-
-You will of course need to have both Storm and OpenSearch installed. For the
-latter, see the [OpenSearch documentation](https://opensearch.org/docs/latest/install-and-configure/install-opensearch/docker/)
-for Docker-based setups.
-
-Schemas are automatically created by the bolts on first use; you can override
-them by providing your own index definitions before starting the topology.
-
-Configuration and dashboards
----------------------
-
-For a ready-to-use crawler configuration, example Flux topologies, index
-initialization scripts and OpenSearch Dashboards exports, refer to the
-[`external/opensearch`](../opensearch) module: all of those resources are
-compatible with this module and have not been duplicated here.
-
-Differences from the legacy `external/opensearch` module
----------------------
-
-* `opensearch.<bolt>.responseBufferSize` is no longer supported. The legacy
-  module used the HC4-based low-level REST client and set a heap response
-  buffer via `HeapBufferedResponseConsumerFactory`. The HC5-based async
-  transport used here does not expose an equivalent per-request override, so
-  the key is ignored. A `WARN` is logged at startup if it is found in the
-  configuration; remove it when migrating.
-* `opensearch.<bolt>.sniff` is no longer supported. The legacy module enabled
-  node auto-discovery by default via the low-level REST client `Sniffer`. The
-  OpenSearch Java Client 3.x does not ship a sniffer equivalent, so this
-  feature is dropped. Keep the `addresses` list up to date manually or put a
-  load balancer in front of the cluster. A `WARN` is logged at startup if the
-  key is found in the configuration; remove it when migrating.
-* Date fields (`nextFetchDate` in the status index and `timestamp` in the
-  metrics index) are serialized as ISO-8601 strings produced by
-  `Instant#toString()` (for example `2026-01-01T00:00:00Z`). The legacy module
-  serializes the same fields through `XContentBuilder#timeField`, which emits
-  ISO-8601 strings via `ISODateTimeFormat.dateTime()` (for example
-  `2026-01-01T00:00:00.000Z`). Both representations are accepted by the
-  default OpenSearch `date` mapping and by the `date_optional_time` /
-  `strict_date_optional_time` formats used in the example mappings under
-  `src/test/resources/`. If you run a custom mapping that restricts the field
-  `format` to `epoch_millis`, update it to accept ISO-8601 (for example
-  `strict_date_optional_time||epoch_millis`) before writing with this module.
diff --git a/external/opensearch/README.md b/external/opensearch/README.md
index b3869c55..48e647a5 100644
--- a/external/opensearch/README.md
+++ b/external/opensearch/README.md
@@ -1,7 +1,9 @@
 stormcrawler-opensearch
 ===========================
 
-A collection of resources for [OpenSearch](https://opensearch.org/):
+A collection of resources for [OpenSearch](https://opensearch.org/) built on the
+[OpenSearch Java Client 3.x](https://opensearch.org/docs/latest/clients/java/) and
+Apache HttpClient 5:
 * [IndexerBolt](https://github.com/apache/stormcrawler/blob/master/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java) for indexing documents crawled with StormCrawler
 * [Spouts](https://github.com/apache/stormcrawler/blob/master/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java) and [StatusUpdaterBolt](https://github.com/apache/stormcrawler/blob/master/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java) for persisting URL information in recursive crawls
 * [MetricsConsumer](https://github.com/apache/stormcrawler/blob/master/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java)
@@ -11,6 +13,25 @@ as well as resources for building basic real-time monitoring dashboards for the
 
 This module is ported from the Elasticsearch one.
 
+The bolts and metrics consumers that write to OpenSearch send their operations
+through `AsyncBulkProcessor`, which groups them into bulk requests. In the
+configuration keys below, `<type>` is `indexer`, `status`, `metrics` or
+`queues`. `AsyncBulkProcessor`:
+
+* sends a bulk request when the number of buffered operations reaches
+  `opensearch.<type>.bulkActions`, when the buffered operations reach 5 MiB of
+  JSON (counted before compression), or when `opensearch.<type>.flushInterval`
+  elapses.
+* resends the items that OpenSearch rejects with status 429 (too many requests)
+  up to 8 times, after 50, 60, 80, 150, 280, 580, 1250 and 2740 ms.
+  `IndexerBolt`, `DeletionBolt` and `StatusUpdaterBolt` fail the tuple of an
+  item still rejected after the last retry.
+* sends the requests without blocking the bolt. The bolt waits only when
+  `opensearch.<type>.concurrentRequests` bulks are in flight, including bulks
+  waiting for a retry.
+* reports the outcome of a bulk to the bolt once, after its last attempt, with
+  the items in request order.
+
 Getting started
 ---------------------
 
@@ -22,7 +43,7 @@ Be sure to replace <CURRENT_VERSION> with the latest released version, which you
 
 You'll be asked to enter a groupId (e.g. com.mycompany.crawler), an artefactId (e.g. stormcrawler), a version, a package name and details about the user agent to use.
 
-This will not only create a fully formed project containing a POM with the dependency above but also a set of resources, configuration files and a topology class. Enter the directory you just created (should be the same as the artefactId you specified earlier) and follow the instructions on the README file.
+This will not only create a fully formed project containing a POM with a dependency on this module but also a set of resources, configuration files and a topology class. Enter the directory you just created (should be the same as the artefactId you specified earlier) and follow the instructions on the README file.
 
 You will of course need to have both Storm and OpenSearch installed. For the latter, the [OpenSearch documentation](https://opensearch.org/docs/latest/install-and-configure/install-opensearch/docker/) contains resources for Docker.
 
@@ -63,10 +84,87 @@ The crawler config YAML must be updated to use an optional argument as shown bel
            argument: "yyyy-MM-dd"
 ```
 
+Upgrading from versions before 4.0
+---------------------
 
-
-
-
-
-
-
+From 4.0, this module uses the OpenSearch Java Client instead of
+`RestHighLevelClient` and `BulkProcessor`.
+
+`stormcrawler-opensearch-java`, released in 3.6.0 and 3.7.0, has the same
+packages, classes and configuration keys. Its users replace that dependency with
+`stormcrawler-opensearch`. For them, the 429 retries and the 5 MiB limit
+described above are new, and `AsyncBulkProcessor.Builder` takes an
+`OpenSearchAsyncClient`. The other changes below apply to users of
+`stormcrawler-opensearch`.
+
+### Configuration
+
+* Earlier versions discovered the cluster nodes by sniffing, which was on by
+  default. This module sends requests only to the nodes in
+  `opensearch.<type>.addresses`. List all nodes there, or put a load balancer in
+  front of the cluster. `opensearch.sniff` and `opensearch.<type>.sniff` are
+  ignored.
+* `opensearch.responseBufferSize` and `opensearch.<type>.responseBufferSize` are
+  no longer read.
+* The bolts and metrics consumers that write through `AsyncBulkProcessor` log a
+  WARN at startup when `opensearch.<type>.sniff` or
+  `opensearch.<type>.responseBufferSize` is set.
+  Remove these keys, and their global forms, from the configuration.
+* `opensearch.<type>.flushInterval` accepts a number followed by `ms`, `s` or
+  `m`, or a number of milliseconds without a unit. Any other value, for example
+  `1h`, logs a WARN and the interval falls back to 5 seconds.
+* `opensearch.<type>.concurrentRequests: 0` behaves as `1`. Earlier versions
+  sent bulk requests synchronously with `0`. This module always sends them
+  asynchronously.
+
+### Behaviour
+
+* If a mapping resource (`status.mapping`, `indexer.mapping`, `metrics.mapping`
+  or `queues.mapping`) is missing from the classpath, the component logs a WARN
+  and starts without creating its index or template. Earlier versions failed at
+  startup. The archetype generates these files under `src/main/resources`.
+* Items rejected with status 429 are resent even when other items of the same
+  bulk failed with another status, for example a 409 from the `create` of a URL
+  that is already in the status index. The `BulkProcessor` of earlier versions
+  retried a bulk only when every failed item had status 429.
+* The 5 MiB limit counts the bytes of JSON sent to OpenSearch. Earlier versions
+  estimated the size from the document sources.
+* `IndexerBolt` fails the tuple when building or sending a document throws any
+  exception. Earlier versions caught only `IOException`, and other exceptions
+  escaped `execute`.
+* When a metadata value indexed by `IndexerBolt` is a single null, the field is
+  left out of the document. Earlier versions wrote it as `null`.
+* `HybridSpout` sends the queries that refill its emptied queues one at a time,
+  on the thread that also runs the aggregation query. Earlier versions sent them
+  in parallel.
+* The dates `nextFetchDate` in the status index and `timestamp` in the metrics
+  index are written with `Instant#toString()`, which leaves out the milliseconds
+  when they are zero (`2026-01-01T00:00:00Z`). Earlier versions always wrote
+  them (`2026-01-01T00:00:00.000Z`). The `date_optional_time` format of the
+  provided mappings accepts both.
+
+### Custom Java code
+
+Custom code that calls or extends these classes must switch to the OpenSearch
+Java Client types:
+
+* `OpenSearchConnection.getClient(conf, type)` returns an `OpenSearchClient`.
+  Close it with `client._transport().close()`.
+* `OpenSearchConnection.getConnection` takes an `AsyncBulkProcessor.Listener`,
+  and `addToProcessor` takes a `BulkOperation`. The listener receives
+  `org.opensearch.client.opensearch.core.BulkRequest` and `BulkResponse`.
+* `AbstractSpout.client` is an `OpenSearchClient`, and `addHitToBuffer` takes
+  the source of a hit as a `Map<String, Object>`.
+* `AggregationSpout` handles the search results in
+  `handleResponse(SearchResponse<JsonData>)`. The sort values passed to
+  `sortValuesForKey` are `FieldValue` objects.
+* `IndexCreation.checkOrCreateIndex` and `checkOrCreateIndexTemplate` take an
+  `OpenSearchClient`. `WaitAckCache.processBulkResponse` and `processFailedBulk`
+  take the client's `BulkResponse` and `BulkRequest`.
+* `BulkItemResponseToFailedFlag` is a record. Read `failed()`, `id()` and
+  `response()` instead of the public fields. `getFailure()` returns a `String`.
+  `status()` and the other `BulkItemResponse` methods are removed. Use
+  `getStatus()`, which returns an `int`, or `response()`.
+* The module no longer depends on the OpenSearch server jar. Code that uses
+  classes such as `QueryBuilders` or `XContentBuilder` must declare that
+  dependency itself or use the Java client equivalents.


(stormcrawler) 02/07: replace the legacy opensearch sources with the opensearch-java ones

Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

dpol1 pushed a commit to branch opensearch-3.x
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git

commit 1ba7227ad600ce8778b64315e0a4bc74ff820743
Author: Davide Polato <dp...@apache.org>
AuthorDate: Fri Oct 16 16:13:49 2026 +0200

    replace the legacy opensearch sources with the opensearch-java ones
    
    The files of external/opensearch/src that also exist in
    external/opensearch-java take the content of their opensearch-java
    counterparts in place, so git blame keeps the lines that did not
    change. The files that exist only in external/opensearch-java are
    moved. The following commits update the module's POM, configuration
    and README.
    
    Signed-off-by: Davide Polato <dp...@apache.org>
---
 .../opensearch/BulkItemResponseToFailedFlag.java   |  71 --
 .../apache/stormcrawler/opensearch/Constants.java  |  23 -
 .../stormcrawler/opensearch/DelegateRefresher.java | 181 ------
 .../stormcrawler/opensearch/IndexCreation.java     | 116 ----
 .../opensearch/OpenSearchConnection.java           | 516 ---------------
 .../stormcrawler/opensearch/WaitAckCache.java      | 340 ----------
 .../stormcrawler/opensearch/bolt/DeletionBolt.java | 180 ------
 .../stormcrawler/opensearch/bolt/IndexerBolt.java  | 297 ---------
 .../opensearch/filtering/JSONURLFilterWrapper.java |  83 ---
 .../opensearch/metrics/MetricsConsumer.java        | 163 -----
 .../opensearch/metrics/MetricsReporter.java        | 219 -------
 .../opensearch/metrics/StatusMetricsBolt.java      | 185 ------
 .../parse/filter/JSONResourceWrapper.java          |  79 ---
 .../opensearch/persistence/AbstractSpout.java      | 330 ----------
 .../opensearch/persistence/AggregationSpout.java   | 432 -------------
 .../opensearch/persistence/HybridSpout.java        | 263 --------
 .../opensearch/persistence/QueueBolt.java          | 191 ------
 .../opensearch/persistence/StatusUpdaterBolt.java  | 358 -----------
 .../opensearch/DelegateRefresherTest.java          | 256 --------
 .../OpenSearchConnectionCredentialsTest.java       | 128 ----
 .../stormcrawler/opensearch/WaitAckCacheTest.java  | 336 ----------
 .../opensearch/bolt/AbstractOpenSearchTest.java    |  53 --
 .../opensearch/bolt/IndexerBoltTest.java           | 138 ----
 .../opensearch/bolt/QueueBoltTest.java             | 202 ------
 .../opensearch/bolt/StatusBoltTest.java            | 176 -----
 .../opensearch/metrics/MetricsReporterTest.java    |  54 --
 .../src/test/resources/indexer.mapping             |  40 --
 .../src/test/resources/metrics.mapping             |  40 --
 .../src/test/resources/queues.mapping              |  33 -
 .../src/test/resources/status.mapping              |  39 --
 .../opensearch/AsyncBulkProcessor.java             |   0
 .../opensearch/BulkItemResponseToFailedFlag.java   | 139 ++--
 .../stormcrawler/opensearch/DelegateRefresher.java |  33 +-
 .../stormcrawler/opensearch/IndexCreation.java     |  84 +--
 .../opensearch/OpenSearchConnection.java           | 715 +++++++++++----------
 .../stormcrawler/opensearch/WaitAckCache.java      |  59 +-
 .../stormcrawler/opensearch/bolt/DeletionBolt.java |  22 +-
 .../stormcrawler/opensearch/bolt/IndexerBolt.java  |  83 ++-
 .../opensearch/filtering/JSONURLFilterWrapper.java |   9 +-
 .../opensearch/metrics/MetricsConsumer.java        |  33 +-
 .../opensearch/metrics/MetricsReporter.java        |  32 +-
 .../opensearch/metrics/StatusMetricsBolt.java      | 135 ++--
 .../parse/filter/JSONResourceWrapper.java          |   9 +-
 .../opensearch/persistence/AbstractSpout.java      | 107 ++-
 .../opensearch/persistence/AggregationSpout.java   | 277 ++++----
 .../opensearch/persistence/HybridSpout.java        | 162 +++--
 .../opensearch/persistence/QueueBolt.java          |  41 +-
 .../opensearch/persistence/StatusUpdaterBolt.java  | 125 ++--
 .../opensearch/AsyncBulkProcessorHttpTest.java     |   0
 .../opensearch/AsyncBulkProcessorRetryTest.java    |   0
 .../opensearch/AsyncBulkProcessorTest.java         |   0
 .../OpenSearchConnectionCredentialsTest.java       | 199 ++----
 .../opensearch/OpenSearchConnectionTest.java       |   0
 .../stormcrawler/opensearch/WaitAckCacheTest.java  | 155 ++---
 .../opensearch/bolt/AbstractOpenSearchTest.java    |   3 +-
 .../opensearch/bolt/NullMetadataValueTest.java     |   0
 .../opensearch/bolt/QueueBoltTest.java             |  45 +-
 .../opensearch/bolt/StatusBoltTest.java            |  41 +-
 .../opensearch/persistence/AbstractSpoutTest.java  |   0
 59 files changed, 1323 insertions(+), 6707 deletions(-)

diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
deleted file mode 100644
index c32f162f..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch;
-
-import java.util.Objects;
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-import org.opensearch.client.opensearch._types.ErrorCause;
-import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
-
-/**
- * Wraps a {@link BulkResponseItem} with a pre-computed failure flag. A 409 (conflict) is not
- * considered a failure — it simply indicates a document already existed when using create mode.
- *
- * @param response the original bulk response item
- * @param failed whether this item represents a real failure (excludes 409 conflicts)
- * @param id the document id from the response item
- */
-public record BulkItemResponseToFailedFlag(
-        @NotNull BulkResponseItem response, boolean failed, @NotNull String id) {
-
-    public BulkItemResponseToFailedFlag {
-        Objects.requireNonNull(response, "response");
-        Objects.requireNonNull(id, "id");
-    }
-
-    /** Constructs with id derived from the response item. */
-    public BulkItemResponseToFailedFlag(@NotNull BulkResponseItem response, boolean failed) {
-        this(
-                response,
-                failed,
-                Objects.requireNonNull(response.id(), "BulkResponseItem id must not be null"));
-    }
-
-    /** Returns the error cause, or {@code null} if the item did not fail. */
-    @Nullable
-    public ErrorCause getFailedCause() {
-        return response.error();
-    }
-
-    /** Returns a human-readable failure description, or {@code null} if the item did not fail. */
-    @Nullable
-    public String getFailure() {
-        ErrorCause error = response.error();
-        if (error == null) {
-            return null;
-        }
-        return error.reason() != null ? error.reason() : error.type();
-    }
-
-    // opensearch-java: status() returns int HTTP code, not RestStatus enum
-    /** Returns the HTTP status code of this response item. */
-    public int getStatus() {
-        return response.status();
-    }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/Constants.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/Constants.java
deleted file mode 100644
index 8c0cbc98..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/Constants.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch;
-
-public interface Constants {
-
-    String PARAMPREFIX = "opensearch.";
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java
deleted file mode 100644
index fb03a9f0..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Map;
-import java.util.Timer;
-import java.util.TimerTask;
-import org.apache.stormcrawler.JSONResource;
-import org.opensearch.client.json.JsonData;
-import org.opensearch.client.opensearch.OpenSearchClient;
-import org.opensearch.client.opensearch.core.GetResponse;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Loads a delegate class that implements both a required base type and {@link JSONResource}, then
- * periodically refreshes its configuration from OpenSearch. Used by {@link
- * org.apache.stormcrawler.opensearch.filtering.JSONURLFilterWrapper} and {@link
- * org.apache.stormcrawler.opensearch.parse.filter.JSONResourceWrapper} to eliminate duplicated
- * setup/refresh/cleanup logic.
- *
- * <p>This is the opensearch-java (OpenSearch Java Client 3.x / HC5) counterpart of the class with
- * the same name in the {@code external/opensearch} module. It uses the typed {@link
- * OpenSearchClient} instead of the deprecated {@code RestHighLevelClient}.
- *
- * @param <T> the base type that the delegate must extend (e.g. URLFilter or ParseFilter)
- */
-public class DelegateRefresher<T> {
-
-    private static final Logger LOG = LoggerFactory.getLogger(DelegateRefresher.class);
-
-    private final T delegate;
-    private Timer refreshTimer;
-    private OpenSearchClient osClient;
-
-    /**
-     * Creates a refresher by loading the delegate class from the JSON configuration.
-     *
-     * @param baseType the required base class (e.g. URLFilter.class or ParseFilter.class)
-     * @param stormConf the Storm configuration map
-     * @param filterParams the JSON params node containing "delegate" and optional "refresh"
-     * @param configurer callback to configure the delegate after instantiation
-     */
-    public DelegateRefresher(
-            Class<T> baseType,
-            Map<String, Object> stormConf,
-            JsonNode filterParams,
-            DelegateConfigure<T> configurer) {
-
-        JsonNode delegateNode = filterParams.get("delegate");
-        if (delegateNode == null) {
-            throw new RuntimeException("delegateNode undefined!");
-        }
-
-        String delegateClassName = null;
-        JsonNode node = delegateNode.get("class");
-        if (node != null && node.isTextual()) {
-            delegateClassName = node.asText();
-        }
-        if (delegateClassName == null) {
-            throw new RuntimeException(baseType.getSimpleName() + " delegate class undefined!");
-        }
-
-        try {
-            Class<?> filterClass = Class.forName(delegateClassName);
-
-            if (!baseType.isAssignableFrom(filterClass)) {
-                throw new RuntimeException(
-                        "Filter " + delegateClassName + " does not extend " + baseType.getName());
-            }
-
-            @SuppressWarnings("unchecked")
-            T instance = (T) filterClass.getDeclaredConstructor().newInstance();
-
-            if (!(instance instanceof JSONResource)) {
-                throw new RuntimeException(
-                        "Filter " + delegateClassName + " does not implement JSONResource");
-            }
-
-            this.delegate = instance;
-        } catch (RuntimeException e) {
-            throw e;
-        } catch (Exception e) {
-            LOG.error("Can't setup {}: {}", delegateClassName, e);
-            throw new RuntimeException("Can't setup " + delegateClassName, e);
-        }
-
-        // configure the delegate
-        JsonNode paramsNode = delegateNode.get("params");
-        configurer.configure(delegate, stormConf, paramsNode);
-
-        // set up periodic refresh from OpenSearch
-        int refreshRate = 600;
-        node = filterParams.get("refresh");
-        if (node != null && (node.isInt() || node.isTextual())) {
-            refreshRate = node.asInt(refreshRate);
-        }
-
-        final JSONResource resource = (JSONResource) delegate;
-
-        refreshTimer = new Timer();
-        refreshTimer.schedule(
-                new TimerTask() {
-                    public void run() {
-                        if (osClient == null) {
-                            try {
-                                osClient = OpenSearchConnection.getClient(stormConf, "config");
-                            } catch (Exception e) {
-                                LOG.error("Exception while creating OpenSearch connection", e);
-                            }
-                        }
-                        if (osClient != null) {
-                            LOG.info("Reloading json resources from OpenSearch");
-                            try {
-                                GetResponse<JsonData> response =
-                                        osClient.get(
-                                                g ->
-                                                        g.index("config")
-                                                                .id(resource.getResourceFile()),
-                                                JsonData.class);
-                                if (response.found() && response.source() != null) {
-                                    String json = response.source().toJson().toString();
-                                    resource.loadJSONResources(
-                                            new ByteArrayInputStream(
-                                                    json.getBytes(StandardCharsets.UTF_8)));
-                                }
-                            } catch (Exception e) {
-                                LOG.error("Can't load config from OpenSearch", e);
-                            }
-                        }
-                    }
-                },
-                refreshRate * 1000L,
-                refreshRate * 1000L);
-    }
-
-    /** Returns the delegate instance. */
-    public T getDelegate() {
-        return delegate;
-    }
-
-    /** Cancels the refresh timer and closes the OpenSearch client. */
-    public void cleanup() {
-        if (refreshTimer != null) {
-            refreshTimer.cancel();
-        }
-        if (osClient != null) {
-            try {
-                osClient._transport().close();
-            } catch (IOException e) {
-                LOG.error("Exception when closing OpenSearch client", e);
-            }
-            osClient = null;
-        }
-    }
-
-    /** Callback interface for configuring the delegate after instantiation. */
-    @FunctionalInterface
-    public interface DelegateConfigure<T> {
-        void configure(T delegate, Map<String, Object> stormConf, JsonNode params);
-    }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java
deleted file mode 100644
index f7617205..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch;
-
-import com.google.common.io.Resources;
-import java.io.IOException;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import org.opensearch.client.opensearch.OpenSearchClient;
-import org.opensearch.client.opensearch.generic.Requests;
-import org.opensearch.client.opensearch.generic.Response;
-import org.opensearch.client.opensearch.indices.ExistsTemplateRequest;
-import org.slf4j.Logger;
-
-public class IndexCreation {
-
-    public static synchronized void checkOrCreateIndex(
-            OpenSearchClient client, String indexName, String boltType, Logger log)
-            throws IOException {
-        final boolean indexExists = client.indices().exists(req -> req.index(indexName)).value();
-        log.info("Index '{}' exists? {}", indexName, indexExists);
-        // there's a possible check-then-update race condition
-        // createIndex intentionally catches and logs exceptions from OpenSearch
-        if (!indexExists) {
-            boolean created =
-                    IndexCreation.createIndex(client, indexName, boltType + ".mapping", log);
-            log.info("Index '{}' created? {} using {}", indexName, created, boltType + ".mapping");
-        }
-    }
-
-    public static synchronized void checkOrCreateIndexTemplate(
-            OpenSearchClient client, String boltType, Logger log) throws IOException {
-        final String templateName = boltType + "-template";
-        final boolean templateExists =
-                client.indices()
-                        .existsTemplate(ExistsTemplateRequest.of(r -> r.name(templateName)))
-                        .value();
-        log.info("Template '{}' exists? {}", templateName, templateExists);
-        // there's a possible check-then-update race condition
-        // createTemplate intentionally catches and logs exceptions from OpenSearch
-        if (!templateExists) {
-            boolean created =
-                    IndexCreation.createTemplate(client, templateName, boltType + ".mapping", log);
-            log.info("templateExists '{}' created? {}", templateName, created);
-        }
-    }
-
-    private static boolean createTemplate(
-            OpenSearchClient client, String templateName, String resourceName, Logger log) {
-
-        try {
-            final URL mapping =
-                    Thread.currentThread().getContextClassLoader().getResource(resourceName);
-
-            final String jsonIndexConfiguration =
-                    Resources.toString(mapping, StandardCharsets.UTF_8);
-
-            try (Response response =
-                    client.generic()
-                            .execute(
-                                    Requests.builder()
-                                            .endpoint("/_template/" + templateName)
-                                            .method("PUT")
-                                            .json(jsonIndexConfiguration)
-                                            .build())) {
-                int statusCode = response.getStatus();
-                return statusCode == 200 || statusCode == 201;
-            }
-        } catch (Exception e) {
-            log.warn("template '{}' not created", templateName, e);
-            return false;
-        }
-    }
-
-    private static boolean createIndex(
-            OpenSearchClient client, String indexName, String resourceName, Logger log) {
-
-        try {
-            final URL mapping =
-                    Thread.currentThread().getContextClassLoader().getResource(resourceName);
-
-            final String jsonIndexConfiguration =
-                    Resources.toString(mapping, StandardCharsets.UTF_8);
-
-            try (Response response =
-                    client.generic()
-                            .execute(
-                                    Requests.builder()
-                                            .endpoint("/" + indexName)
-                                            .method("PUT")
-                                            .json(jsonIndexConfiguration)
-                                            .build())) {
-                int statusCode = response.getStatus();
-                return statusCode == 200 || statusCode == 201;
-            }
-        } catch (Exception e) {
-            log.warn("index '{}' not created", indexName, e);
-            return false;
-        }
-    }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
deleted file mode 100644
index eeb47362..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
+++ /dev/null
@@ -1,516 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch;
-
-import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_CONNECT_TIMEOUT_MILLIS;
-import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_RESPONSE_TIMEOUT_MILLIS;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.regex.Pattern;
-import javax.net.ssl.SSLContext;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hc.client5.http.auth.AuthScope;
-import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
-import org.apache.hc.client5.http.config.ConnectionConfig;
-import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
-import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
-import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
-import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
-import org.apache.hc.core5.http.HttpHost;
-import org.apache.hc.core5.reactor.ssl.TlsDetails;
-import org.apache.hc.core5.ssl.SSLContextBuilder;
-import org.apache.hc.core5.util.Timeout;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.jetbrains.annotations.NotNull;
-import org.opensearch.client.json.jackson.JacksonJsonpMapper;
-import org.opensearch.client.opensearch.OpenSearchAsyncClient;
-import org.opensearch.client.opensearch.OpenSearchClient;
-import org.opensearch.client.opensearch.core.bulk.BulkOperation;
-import org.opensearch.client.transport.OpenSearchTransport;
-import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Utility class to instantiate an OpenSearch client and bulkprocessor based on the configuration.
- */
-public final class OpenSearchConnection {
-
-    private static final Logger LOG = LoggerFactory.getLogger(OpenSearchConnection.class);
-
-    @NotNull private final OpenSearchClient client;
-
-    @NotNull private final AsyncBulkProcessor processor;
-
-    @NotNull private final OpenSearchTransport transport;
-
-    private OpenSearchConnection(
-            @NotNull OpenSearchClient c,
-            @NotNull AsyncBulkProcessor p,
-            @NotNull OpenSearchTransport t) {
-        client = c;
-        processor = p;
-        transport = t;
-    }
-
-    public OpenSearchClient getClient() {
-        return client;
-    }
-
-    /**
-     * Creates a standalone {@link OpenSearchClient}. Used by classes that need a client without a
-     * bulk processor (e.g. spouts, filters). Callers are responsible for closing the returned
-     * client's transport via {@code client._transport().close()}.
-     */
-    public static OpenSearchClient getClient(Map<String, Object> stormConf, String boltType) {
-        return buildClientResources(stormConf, boltType).client();
-    }
-
-    /** Adds a single bulk operation to the internal processor. */
-    public void addToProcessor(final BulkOperation operation) {
-        processor.add(operation);
-    }
-
-    /**
-     * Creates a connection with a default (no-op) listener. The values for bolt type are
-     * [indexer,status,metrics].
-     */
-    public static OpenSearchConnection getConnection(
-            Map<String, Object> stormConf, String boltType) {
-        AsyncBulkProcessor.Listener listener =
-                new AsyncBulkProcessor.Listener() {
-                    @Override
-                    public void afterBulk(
-                            long arg0,
-                            org.opensearch.client.opensearch.core.BulkRequest arg1,
-                            org.opensearch.client.opensearch.core.BulkResponse arg2) {}
-
-                    @Override
-                    public void afterBulk(
-                            long arg0,
-                            org.opensearch.client.opensearch.core.BulkRequest arg1,
-                            Throwable arg2) {}
-
-                    @Override
-                    public void beforeBulk(
-                            long arg0, org.opensearch.client.opensearch.core.BulkRequest arg1) {}
-                };
-        return getConnection(stormConf, boltType, listener);
-    }
-
-    public static OpenSearchConnection getConnection(
-            Map<String, Object> stormConf, String boltType, AsyncBulkProcessor.Listener listener) {
-
-        final String dottedType = boltType + ".";
-
-        warnOnRemovedKeys(stormConf, dottedType);
-
-        ClientResources cr = buildClientResources(stormConf, boltType);
-
-        final String flushIntervalString =
-                ConfUtils.getString(
-                        stormConf, Constants.PARAMPREFIX, dottedType, "flushInterval", "5s");
-
-        final long flushIntervalMillis = parseTimeValueToMillis(flushIntervalString, 5000);
-
-        final int bulkActions =
-                ConfUtils.getInt(stormConf, Constants.PARAMPREFIX, dottedType, "bulkActions", 50);
-
-        final int concurrentRequests =
-                ConfUtils.getInt(
-                        stormConf, Constants.PARAMPREFIX, dottedType, "concurrentRequests", 1);
-
-        AsyncBulkProcessor bulkProcessor = null;
-        try {
-            bulkProcessor =
-                    new AsyncBulkProcessor.Builder(
-                                    new OpenSearchAsyncClient(cr.transport()), listener)
-                            .setBulkActions(bulkActions)
-                            .setFlushIntervalMillis(flushIntervalMillis)
-                            .setConcurrentRequests(concurrentRequests)
-                            .build();
-
-            return new OpenSearchConnection(cr.client(), bulkProcessor, cr.transport());
-        } catch (Exception e) {
-            if (bulkProcessor != null) {
-                try {
-                    bulkProcessor.close();
-                } catch (Exception suppressed) {
-                    e.addSuppressed(suppressed);
-                }
-            }
-            try {
-                cr.transport().close();
-            } catch (IOException suppressed) {
-                e.addSuppressed(suppressed);
-            }
-            throw e;
-        }
-    }
-
-    private final AtomicBoolean isClosed = new AtomicBoolean(false);
-
-    public void close() {
-
-        if (!isClosed.compareAndSet(false, true)) {
-            LOG.warn("Tried to close an already closed connection!");
-            return;
-        }
-
-        LOG.debug("Start closing the OpenSearch connection");
-
-        // First, close the BulkProcessor ensuring pending actions are flushed
-        try {
-            boolean success = processor.awaitClose(60, TimeUnit.SECONDS);
-            if (!success) {
-                throw new RuntimeException(
-                        "Failed to flush pending actions when closing BulkProcessor");
-            }
-        } catch (InterruptedException e) {
-            throw new RuntimeException(e);
-        }
-
-        // Now close the transport (also shuts down the underlying HTTP client)
-        try {
-            transport.close();
-        } catch (IOException e) {
-            LOG.trace("Transport threw IO exception on close.");
-        }
-    }
-
-    /**
-     * Extracts the document ID from a {@link BulkOperation} regardless of its type (index, create,
-     * delete, update).
-     */
-    public static String getBulkOperationId(BulkOperation op) {
-        if (op.isIndex()) {
-            return op.index().id();
-        }
-        if (op.isCreate()) {
-            return op.create().id();
-        }
-        if (op.isDelete()) {
-            return op.delete().id();
-        }
-        if (op.isUpdate()) {
-            return op.update().id();
-        }
-        return null;
-    }
-
-    /**
-     * Returns the scopes the Basic credentials are registered for: the scheme, host and port of
-     * each configured address. A request to any other address does not receive them.
-     */
-    static List<AuthScope> credentialScopes(List<HttpHost> hosts) {
-        final Set<AuthScope> scopes = new LinkedHashSet<>();
-        for (HttpHost host : hosts) {
-            scopes.add(new AuthScope(host));
-        }
-        return new ArrayList<>(scopes);
-    }
-
-    /**
-     * Returns the addresses which use plain http and are not a loopback address, i.e. those the
-     * Basic credentials would be sent to in the clear over the network.
-     */
-    static List<HttpHost> plainHttpHosts(List<HttpHost> hosts) {
-        final List<HttpHost> plain = new ArrayList<>();
-        for (HttpHost host : hosts) {
-            if ("http".equalsIgnoreCase(host.getSchemeName()) && !isLoopback(host.getHostName())) {
-                plain.add(host);
-            }
-        }
-        return plain;
-    }
-
-    private static final Pattern IPV4_LOOPBACK = Pattern.compile("127(\\.\\d{1,3}){3}");
-
-    /** Whether the host is localhost or a loopback IP literal. No name is resolved. */
-    static boolean isLoopback(String hostname) {
-        String host = StringUtils.strip(hostname.toLowerCase(Locale.ROOT), "[]");
-        return host.equals("localhost")
-                || IPV4_LOOPBACK.matcher(host).matches()
-                || host.equals("::1")
-                || host.equals("0:0:0:0:0:0:0:1");
-    }
-
-    private static void warnAboutPlainHttp(String boltType, List<HttpHost> hosts) {
-        final List<HttpHost> plain = plainHttpHosts(hosts);
-        if (!plain.isEmpty()) {
-            LOG.warn(
-                    "OpenSearch credentials are configured for {} but the addresses {} use plain "
-                            + "http, the credentials are sent unencrypted. Give the addresses an "
-                            + "https:// scheme.",
-                    boltType,
-                    plain);
-        }
-    }
-
-    // internal helpers
-    private record ClientResources(OpenSearchClient client, OpenSearchTransport transport) {}
-
-    /**
-     * Logs a WARN for legacy configuration keys that are no longer honoured by this module, so that
-     * users migrating from {@code external/opensearch} notice silently-dropped tuning. See the
-     * module README for the full list of differences.
-     */
-    private static void warnOnRemovedKeys(Map<String, Object> stormConf, String dottedType) {
-        final String responseBufferKey = Constants.PARAMPREFIX + dottedType + "responseBufferSize";
-        if (stormConf.containsKey(responseBufferKey)) {
-            LOG.warn(
-                    "Configuration key '{}' is set but no longer supported by the opensearch-java module. "
-                            + "The HC5-based async transport does not expose an equivalent per-request "
-                            + "heap-buffer override. The setting is ignored — remove it from your "
-                            + "configuration. See external/opensearch-java/README.md for details.",
-                    responseBufferKey);
-        }
-
-        final String sniffKey = Constants.PARAMPREFIX + dottedType + "sniff";
-        if (stormConf.containsKey(sniffKey)) {
-            LOG.warn(
-                    "Configuration key '{}' is set but no longer supported by the opensearch-java module. "
-                            + "The OpenSearch Java Client 3.x does not ship a Sniffer equivalent, so "
-                            + "automatic node discovery is not available. Keep the 'addresses' list up to "
-                            + "date manually or put a load balancer in front of the cluster. "
-                            + "See external/opensearch-java/README.md for details.",
-                    sniffKey);
-        }
-    }
-
-    private static ClientResources buildClientResources(
-            Map<String, Object> stormConf, String boltType) {
-
-        final String dottedType = boltType + ".";
-
-        final List<HttpHost> hosts = new ArrayList<>();
-
-        final List<String> confighosts =
-                ConfUtils.loadListFromConf(
-                        Constants.PARAMPREFIX, dottedType, "addresses", stormConf);
-
-        // find ; separated values and tokenise as multiple addresses
-        // e.g. opensearch1:9200; opensearch2:9200
-        if (confighosts.size() == 1) {
-            String input = confighosts.get(0);
-            confighosts.clear();
-            confighosts.addAll(Arrays.asList(input.split(" *; *")));
-        }
-
-        for (String host : confighosts) {
-            // no port specified? use default one
-            int port = 9200;
-            String scheme = "http";
-            // no scheme specified? use http
-            if (!host.startsWith(scheme)) {
-                host = "http://" + host;
-            }
-            URI uri = URI.create(host);
-            if (uri.getHost() == null) {
-                throw new RuntimeException("host undefined " + host);
-            }
-            if (uri.getPort() != -1) {
-                port = uri.getPort();
-            }
-            if (uri.getScheme() != null) {
-                scheme = uri.getScheme();
-            }
-            // HC5: constructor is (scheme, hostname, port) — not (hostname, port, scheme)
-            hosts.add(new HttpHost(scheme, uri.getHost(), port));
-        }
-
-        LOG.info(
-                "OpenSearch {} transport configured with {} host(s): {}",
-                boltType,
-                hosts.size(),
-                hosts);
-
-        // authentication via user / password
-        final String user =
-                ConfUtils.getString(stormConf, Constants.PARAMPREFIX, dottedType, "user");
-        final String password =
-                ConfUtils.getString(stormConf, Constants.PARAMPREFIX, dottedType, "password");
-
-        final String proxyhost =
-                ConfUtils.getString(stormConf, Constants.PARAMPREFIX, dottedType, "proxy.host");
-
-        final int proxyport =
-                ConfUtils.getInt(stormConf, Constants.PARAMPREFIX, dottedType, "proxy.port", -1);
-
-        final String proxyscheme =
-                ConfUtils.getString(
-                        stormConf, Constants.PARAMPREFIX, dottedType, "proxy.scheme", "http");
-
-        final boolean disableTlsValidation =
-                ConfUtils.getBoolean(
-                        stormConf, Constants.PARAMPREFIX, "", "disable.tls.validation", false);
-
-        final boolean needsUser = StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password);
-        final boolean needsProxy = StringUtils.isNotBlank(proxyhost) && proxyport != -1;
-
-        // Defaults from ApacheHttpClient5TransportBuilder (same as the former RestClientBuilder)
-        final int connectTimeout =
-                ConfUtils.getInt(
-                        stormConf,
-                        Constants.PARAMPREFIX,
-                        dottedType,
-                        "connect.timeout",
-                        DEFAULT_CONNECT_TIMEOUT_MILLIS);
-        final int socketTimeout =
-                ConfUtils.getInt(
-                        stormConf,
-                        Constants.PARAMPREFIX,
-                        dottedType,
-                        "socket.timeout",
-                        DEFAULT_RESPONSE_TIMEOUT_MILLIS);
-
-        final boolean compression =
-                ConfUtils.getBoolean(
-                        stormConf, Constants.PARAMPREFIX, dottedType, "compression", false);
-
-        final ApacheHttpClient5TransportBuilder builder =
-                ApacheHttpClient5TransportBuilder.builder(hosts.toArray(new HttpHost[0]))
-                        .setMapper(new JacksonJsonpMapper());
-
-        // Timeouts via ConnectionConfig on the builder's internal connection manager
-        builder.setConnectionConfigCallback(
-                connConfigBuilder ->
-                        connConfigBuilder
-                                .setConnectTimeout(Timeout.ofMilliseconds(connectTimeout))
-                                .setSocketTimeout(Timeout.ofMilliseconds(socketTimeout)));
-
-        if (needsUser) {
-            warnAboutPlainHttp(boltType, hosts);
-        }
-        if (disableTlsValidation) {
-            LOG.warn(
-                    "opensearch.disable.tls.validation is set: the certificates and host names of "
-                            + "the OpenSearch nodes used for {} are not checked",
-                    boltType);
-        }
-
-        // Auth, proxy, and/or trust-all SSL via HttpClient customisation
-        if (needsUser || needsProxy || disableTlsValidation) {
-            builder.setHttpClientConfigCallback(
-                    httpClientBuilder -> {
-                        // hc.client5 auth: password is char[], AuthScope(host, port)
-                        if (needsUser) {
-                            final BasicCredentialsProvider credentialsProvider =
-                                    new BasicCredentialsProvider();
-                            final UsernamePasswordCredentials credentials =
-                                    new UsernamePasswordCredentials(user, password.toCharArray());
-                            for (AuthScope scope : credentialScopes(hosts)) {
-                                credentialsProvider.setCredentials(scope, credentials);
-                            }
-                            httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
-                        }
-                        // hc.client5 proxy: HttpHost(scheme, host, port)
-                        if (needsProxy) {
-                            httpClientBuilder.setProxy(
-                                    new HttpHost(proxyscheme, proxyhost, proxyport));
-                        }
-                        // Custom connection manager overrides the builder's internal one,
-                        // so timeouts and TlsDetailsFactory must be replicated here
-                        if (disableTlsValidation) {
-                            try {
-                                final SSLContext sslContext =
-                                        SSLContextBuilder.create()
-                                                .loadTrustMaterial((chain, authType) -> true)
-                                                .build();
-                                httpClientBuilder.setConnectionManager(
-                                        PoolingAsyncClientConnectionManagerBuilder.create()
-                                                .setTlsStrategy(
-                                                        ClientTlsStrategyBuilder.create()
-                                                                .setSslContext(sslContext)
-                                                                .setHostnameVerifier(
-                                                                        NoopHostnameVerifier
-                                                                                .INSTANCE)
-                                                                // HTTP/2 ALPN negotiation
-                                                                .setTlsDetailsFactory(
-                                                                        sslEngine ->
-                                                                                new TlsDetails(
-                                                                                        sslEngine
-                                                                                                .getSession(),
-                                                                                        sslEngine
-                                                                                                .getApplicationProtocol()))
-                                                                .build())
-                                                .setDefaultConnectionConfig(
-                                                        ConnectionConfig.custom()
-                                                                .setConnectTimeout(
-                                                                        Timeout.ofMilliseconds(
-                                                                                connectTimeout))
-                                                                .setSocketTimeout(
-                                                                        Timeout.ofMilliseconds(
-                                                                                socketTimeout))
-                                                                .build())
-                                                .build());
-                            } catch (Exception e) {
-                                throw new RuntimeException("Failed to disable TLS validation", e);
-                            }
-                        }
-                        return httpClientBuilder;
-                    });
-        }
-
-        // Compression: first-class builder method, not a request interceptor
-        builder.setCompressionEnabled(compression);
-
-        final OpenSearchTransport transport = builder.build();
-        final OpenSearchClient openSearchClient = new OpenSearchClient(transport);
-
-        return new ClientResources(openSearchClient, transport);
-    }
-
-    /**
-     * Parses a time value string (e.g. "5s", "500ms", "1m") into milliseconds.
-     *
-     * @param value the string to parse
-     * @param defaultMillis the default if parsing fails
-     * @return milliseconds
-     */
-    static long parseTimeValueToMillis(String value, long defaultMillis) {
-        if (value == null || value.isBlank()) {
-            return defaultMillis;
-        }
-        value = value.trim();
-        try {
-            if (value.endsWith("ms")) {
-                return Long.parseLong(value.substring(0, value.length() - 2));
-            } else if (value.endsWith("s")) {
-                return Long.parseLong(value.substring(0, value.length() - 1)) * 1000;
-            } else if (value.endsWith("m")) {
-                return Long.parseLong(value.substring(0, value.length() - 1)) * 60000;
-            } else {
-                return Long.parseLong(value);
-            }
-        } catch (NumberFormatException e) {
-            LOG.warn("Could not parse time value '{}', using default {}ms", value, defaultMillis);
-            return defaultMillis;
-        }
-    }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java
deleted file mode 100644
index 0e8574fb..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java
+++ /dev/null
@@ -1,340 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch;
-
-import com.github.benmanes.caffeine.cache.Cache;
-import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.RemovalCause;
-import com.github.benmanes.caffeine.cache.Ticker;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
-import org.apache.storm.tuple.Tuple;
-import org.apache.stormcrawler.metrics.ScopedCounter;
-import org.jetbrains.annotations.Nullable;
-// opensearch-java: uses typed BulkRequest/BulkResponse, not legacy REST equivalents
-import org.opensearch.client.opensearch.core.BulkRequest;
-import org.opensearch.client.opensearch.core.BulkResponse;
-import org.slf4j.Logger;
-
-/**
- * Thread-safe cache that tracks in-flight tuples awaiting bulk acknowledgment from OpenSearch.
- * Provides shared logic for processing bulk responses and failing tuples on error, used by
- * IndexerBolt, DeletionBolt, and StatusUpdaterBolt.
- */
-public class WaitAckCache {
-
-    /** Callback invoked for each tuple when processing a successful bulk response. */
-    @FunctionalInterface
-    public interface TupleAction {
-        void handle(String id, Tuple tuple, BulkItemResponseToFailedFlag selected);
-    }
-
-    private final Cache<String, List<Tuple>> cache;
-    private final ReentrantLock lock = new ReentrantLock(true);
-    private final Logger log;
-    private final Consumer<Tuple> onEviction;
-
-    /** Creates a cache with a fixed 60-second expiry. */
-    public WaitAckCache(Logger log, Consumer<Tuple> onEviction) {
-        this(Caffeine.newBuilder().expireAfterWrite(60, TimeUnit.SECONDS), log, onEviction);
-    }
-
-    /**
-     * Creates a cache using a {@link Caffeine} spec string (e.g. {@code "expireAfterWrite=300s"}),
-     * typically driven by {@code topology.message.timeout.secs}.
-     */
-    public WaitAckCache(String cacheSpec, Logger log, Consumer<Tuple> onEviction) {
-        this(Caffeine.from(cacheSpec), log, onEviction);
-    }
-
-    /** Creates a cache with a custom ticker for deterministic time control in tests. */
-    WaitAckCache(String cacheSpec, Logger log, Consumer<Tuple> onEviction, Ticker ticker) {
-        this(Caffeine.from(cacheSpec).ticker(ticker).executor(Runnable::run), log, onEviction);
-    }
-
-    private WaitAckCache(Caffeine<Object, Object> builder, Logger log, Consumer<Tuple> onEviction) {
-        this.log = log;
-        this.onEviction = onEviction;
-        this.cache =
-                builder.<String, List<Tuple>>removalListener(
-                                (String key, List<Tuple> value, RemovalCause cause) -> {
-                                    if (!cause.wasEvicted()) {
-                                        return;
-                                    }
-                                    if (value != null) {
-                                        log.error(
-                                                "Purged from waitAck {} with {} values",
-                                                key,
-                                                value.size());
-                                        for (Tuple t : value) {
-                                            onEviction.accept(t);
-                                        }
-                                    } else {
-                                        log.error("Purged from waitAck {} with no values", key);
-                                    }
-                                })
-                        .build();
-    }
-
-    /** Returns the approximate number of entries in this cache. */
-    public long estimatedSize() {
-        return cache.estimatedSize();
-    }
-
-    /** Adds a tuple to the cache under the given document ID, creating the list if needed. */
-    public void addTuple(String docID, Tuple tuple) {
-        lock.lock();
-        try {
-            List<Tuple> tt = cache.get(docID, k -> new LinkedList<>());
-            tt.add(tuple);
-            if (log.isDebugEnabled()) {
-                String url = (String) tuple.getValueByField("url");
-                log.debug("Added to waitAck {} with ID {} total {}", url, docID, tt.size());
-            }
-        } finally {
-            lock.unlock();
-        }
-    }
-
-    /** Returns true if the cache contains an entry for the given document ID. */
-    public boolean contains(String docID) {
-        lock.lock();
-        try {
-            return cache.getIfPresent(docID) != null;
-        } finally {
-            lock.unlock();
-        }
-    }
-
-    /** Forces pending cache maintenance, triggering eviction listeners for expired entries. */
-    public void cleanUp() {
-        cache.cleanUp();
-    }
-
-    /** Fails all remaining tuples in the cache and invalidates all entries. */
-    public void shutdown() {
-        lock.lock();
-        try {
-            Map<String, List<Tuple>> remaining = cache.asMap();
-            for (var entry : remaining.entrySet()) {
-                log.warn(
-                        "Shutdown: failing {} tuple(s) for ID {}",
-                        entry.getValue().size(),
-                        entry.getKey());
-                for (Tuple t : entry.getValue()) {
-                    onEviction.accept(t);
-                }
-            }
-            cache.invalidateAll();
-        } finally {
-            lock.unlock();
-        }
-    }
-
-    /** Invalidates a single cache entry. */
-    public void invalidate(String docID) {
-        lock.lock();
-        try {
-            cache.invalidate(docID);
-        } finally {
-            lock.unlock();
-        }
-    }
-
-    /**
-     * Processes a successful bulk response: classifies each item (conflict vs failure), retrieves
-     * cached tuples, selects the best response per document ID, and invokes the action for each
-     * tuple.
-     *
-     * @param conflictCounter optional metric counter; if non-null, increments "doc_conflicts" scope
-     *     for each conflict
-     */
-    public void processBulkResponse(
-            BulkResponse response,
-            long executionId,
-            @Nullable ScopedCounter conflictCounter,
-            TupleAction action) {
-
-        // opensearch-java: items() returns List<BulkResponseItem>; status() returns int
-        var idsToBulkItems =
-                response.items().stream()
-                        .map(
-                                bir -> {
-                                    var error = bir.error();
-                                    boolean failed = false;
-                                    if (error != null) {
-                                        // opensearch-java: int status code, not RestStatus enum
-                                        if (bir.status() == 409) {
-                                            if (conflictCounter != null) {
-                                                conflictCounter.scope("doc_conflicts").incrBy(1);
-                                            }
-                                            log.debug("Doc conflict ID {}", bir.id());
-                                        } else {
-                                            log.error(
-                                                    "Bulk item failure ID {}: {}",
-                                                    bir.id(),
-                                                    error.reason() != null
-                                                            ? error.reason()
-                                                            : error.type());
-                                            failed = true;
-                                        }
-                                    }
-                                    return new BulkItemResponseToFailedFlag(bir, failed);
-                                })
-                        .collect(
-                                // https://github.com/apache/stormcrawler/issues/832
-                                Collectors.groupingBy(
-                                        BulkItemResponseToFailedFlag::id,
-                                        Collectors.toUnmodifiableList()));
-
-        Map<String, List<Tuple>> presentTuples;
-        long estimatedSize;
-        Set<String> debugInfo = null;
-        lock.lock();
-        try {
-            presentTuples = cache.getAllPresent(idsToBulkItems.keySet());
-            if (!presentTuples.isEmpty()) {
-                cache.invalidateAll(presentTuples.keySet());
-            }
-            estimatedSize = cache.estimatedSize();
-            if (log.isDebugEnabled() && estimatedSize > 0L) {
-                debugInfo = new HashSet<>(cache.asMap().keySet());
-            }
-        } finally {
-            lock.unlock();
-        }
-
-        int ackCount = 0;
-        int failureCount = 0;
-
-        for (var entry : presentTuples.entrySet()) {
-            final var id = entry.getKey();
-            final var tuples = entry.getValue();
-            final var bulkItems = idsToBulkItems.get(id);
-
-            BulkItemResponseToFailedFlag selected = selectBest(bulkItems, id);
-
-            if (tuples != null) {
-                log.debug("Found {} tuple(s) for ID {}", tuples.size(), id);
-                for (Tuple t : tuples) {
-                    if (selected.failed()) {
-                        failureCount++;
-                    } else {
-                        ackCount++;
-                    }
-                    action.handle(id, t, selected);
-                }
-            } else {
-                log.warn("Could not find unacked tuples for {}", id);
-            }
-        }
-
-        log.info(
-                "Bulk response [{}] : items {}, waitAck {}, acked {}, failed {}",
-                executionId,
-                idsToBulkItems.size(),
-                estimatedSize,
-                ackCount,
-                failureCount);
-
-        if (debugInfo != null) {
-            for (String k : debugInfo) {
-                log.debug("Still in wait ack after bulk response [{}] => {}", executionId, k);
-            }
-        }
-    }
-
-    /**
-     * Processes a failed bulk request by failing all associated tuples.
-     *
-     * @param failAction callback applied to each tuple that must be failed
-     */
-    public void processFailedBulk(
-            BulkRequest request, long executionId, Throwable failure, Consumer<Tuple> failAction) {
-
-        log.error("Exception with bulk {} - failing the whole lot ", executionId, failure);
-
-        // opensearch-java: operations() + getBulkOperationId replaces
-        //   legacy requests() + DocWriteRequest::id
-        final var failedIds =
-                request.operations().stream()
-                        .map(OpenSearchConnection::getBulkOperationId)
-                        .filter(Objects::nonNull)
-                        .collect(Collectors.toUnmodifiableSet());
-
-        Map<String, List<Tuple>> failedTupleLists;
-        lock.lock();
-        try {
-            failedTupleLists = cache.getAllPresent(failedIds);
-            if (!failedTupleLists.isEmpty()) {
-                cache.invalidateAll(failedTupleLists.keySet());
-            }
-        } finally {
-            lock.unlock();
-        }
-
-        for (var id : failedIds) {
-            var tuples = failedTupleLists.get(id);
-            if (tuples != null) {
-                log.debug("Failed {} tuple(s) for ID {}", tuples.size(), id);
-                for (Tuple t : tuples) {
-                    failAction.accept(t);
-                }
-            } else {
-                log.warn("Could not find unacked tuple for {}", id);
-            }
-        }
-    }
-
-    /**
-     * Selects the best response when there are multiple bulk items for the same document ID.
-     * Prefers non-failed responses; warns when there is a mix of success and failure. If all items
-     * are failed, returns the first one (no warning logged since there is no ambiguity).
-     */
-    private BulkItemResponseToFailedFlag selectBest(
-            List<BulkItemResponseToFailedFlag> items, String id) {
-        if (items.size() == 1) {
-            return items.get(0);
-        }
-
-        BulkItemResponseToFailedFlag best = items.get(0);
-        int failedCount = 0;
-        for (var item : items) {
-            if (item.failed()) {
-                failedCount++;
-            } else {
-                best = item;
-            }
-        }
-        if (failedCount > 0 && failedCount < items.size()) {
-            log.warn(
-                    "The id {} would result in an ack and a failure."
-                            + " Using only the ack for processing.",
-                    id);
-        }
-        return best;
-    }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
deleted file mode 100644
index f77662ea..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.bolt;
-
-import java.lang.invoke.MethodHandles;
-import java.util.Map;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.topology.OutputFieldsDeclarer;
-import org.apache.storm.topology.base.BaseRichBolt;
-import org.apache.storm.tuple.Tuple;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
-import org.apache.stormcrawler.metrics.CrawlerMetrics;
-import org.apache.stormcrawler.opensearch.AsyncBulkProcessor;
-import org.apache.stormcrawler.opensearch.OpenSearchConnection;
-import org.apache.stormcrawler.opensearch.WaitAckCache;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.opensearch.core.BulkRequest;
-import org.opensearch.client.opensearch.core.BulkResponse;
-import org.opensearch.client.opensearch.core.bulk.BulkOperation;
-import org.slf4j.LoggerFactory;
-
-/**
- * Deletes documents in OpenSearch. This should be connected to the StatusUpdaterBolt via the
- * 'deletion' stream and will remove the documents with a status of ERROR. Note that this component
- * will also try to delete documents even though they were never indexed and it currently won't
- * delete documents which were indexed under the canonical URL.
- */
-public class DeletionBolt extends BaseRichBolt implements AsyncBulkProcessor.Listener {
-
-    static final org.slf4j.Logger LOG =
-            LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
-    private static final String BOLT_TYPE = "indexer";
-
-    private OutputCollector _collector;
-
-    private String indexName;
-
-    private OpenSearchConnection connection;
-
-    private WaitAckCache waitAck;
-
-    private String docIdMetadataKey;
-
-    public DeletionBolt() {}
-
-    /** Sets the index name instead of taking it from the configuration. * */
-    public DeletionBolt(String indexName) {
-        this.indexName = indexName;
-    }
-
-    @Override
-    public void prepare(
-            Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
-        _collector = collector;
-        if (indexName == null) {
-            indexName = ConfUtils.getString(conf, IndexerBolt.OSIndexNameParamName, "content");
-        }
-
-        docIdMetadataKey =
-                ConfUtils.getString(conf, AbstractIndexerBolt.DOC_ID_METADATA_PARAM_NAME);
-
-        try {
-            connection = OpenSearchConnection.getConnection(conf, BOLT_TYPE, this);
-        } catch (Exception e1) {
-            LOG.error("Can't connect to OpenSearch", e1);
-            throw new RuntimeException(e1);
-        }
-
-        waitAck = new WaitAckCache(LOG, _collector::fail);
-        CrawlerMetrics.registerGauge(context, conf, "waitAck", waitAck::estimatedSize, 10);
-    }
-
-    @Override
-    public void cleanup() {
-        waitAck.shutdown();
-        if (connection != null) {
-            connection.close();
-        }
-    }
-
-    @Override
-    public void execute(Tuple tuple) {
-        String url = tuple.getStringByField("url");
-        Metadata metadata = (Metadata) tuple.getValueByField("metadata");
-
-        // keep it simple for now and ignore cases where the canonical URL was
-        // used
-
-        final String docID = getDocumentID(metadata, url);
-        final String targetIndex = getIndexName(metadata);
-        BulkOperation op = BulkOperation.of(b -> b.delete(d -> d.index(targetIndex).id(docID)));
-
-        waitAck.addTuple(docID, tuple);
-
-        connection.addToProcessor(op);
-    }
-
-    @Override
-    public void declareOutputFields(OutputFieldsDeclarer arg0) {
-        // none
-    }
-
-    /**
-     * Must be overridden for implementing custom index names based on some metadata information By
-     * Default, indexName coming from config is used
-     */
-    protected String getIndexName(Metadata m) {
-        return indexName;
-    }
-
-    /**
-     * Get the document id. Uses the SHA-256 digest of the value found in the metadata for the key
-     * configured with {@link AbstractIndexerBolt#DOC_ID_METADATA_PARAM_NAME} if present, so that
-     * the ID matches the one generated by the indexer for the same document.
-     *
-     * @param metadata The {@link Metadata}.
-     * @param url The normalised url.
-     * @return Return the SHA-256 digest of the value found in the metadata, or of the normalised
-     *     url if not present.
-     */
-    protected String getDocumentID(Metadata metadata, String url) {
-        String source = url;
-        if (docIdMetadataKey != null) {
-            final String fromMetadata =
-                    StringUtils.trimToNull(metadata.getFirstValue(docIdMetadataKey));
-            if (fromMetadata != null) {
-                source = fromMetadata;
-            }
-        }
-        return org.apache.commons.codec.digest.DigestUtils.sha256Hex(source);
-    }
-
-    @Override
-    public void beforeBulk(long executionId, BulkRequest request) {}
-
-    @Override
-    public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
-        waitAck.processBulkResponse(
-                response,
-                executionId,
-                null,
-                (id, t, selected) -> {
-                    if (!selected.failed()) {
-                        _collector.ack(t);
-                    } else {
-                        String url = (String) t.getValueByField("url");
-                        LOG.error(
-                                "update ID {}, URL {}, failure: {}",
-                                id,
-                                url,
-                                selected.getFailure());
-                        _collector.fail(t);
-                    }
-                });
-    }
-
-    @Override
-    public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
-        waitAck.processFailedBulk(request, executionId, failure, _collector::fail);
-    }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
deleted file mode 100644
index f7a44185..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.bolt;
-
-import static org.apache.stormcrawler.Constants.StatusStreamName;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.tuple.Values;
-import org.apache.stormcrawler.Constants;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
-import org.apache.stormcrawler.metrics.CrawlerMetrics;
-import org.apache.stormcrawler.metrics.ScopedCounter;
-import org.apache.stormcrawler.metrics.ScopedReducedMetric;
-import org.apache.stormcrawler.opensearch.AsyncBulkProcessor;
-import org.apache.stormcrawler.opensearch.IndexCreation;
-import org.apache.stormcrawler.opensearch.OpenSearchConnection;
-import org.apache.stormcrawler.opensearch.WaitAckCache;
-import org.apache.stormcrawler.persistence.Status;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.opensearch.core.BulkRequest;
-import org.opensearch.client.opensearch.core.BulkResponse;
-import org.opensearch.client.opensearch.core.bulk.BulkOperation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Sends documents to opensearch. Indexes all the fields from the tuples or a Map
- * &lt;String,Object&gt; from a named field.
- */
-public class IndexerBolt extends AbstractIndexerBolt implements AsyncBulkProcessor.Listener {
-
-    private static final Logger LOG = LoggerFactory.getLogger(IndexerBolt.class);
-
-    private static final String OSBoltType = "indexer";
-
-    static final String OSIndexNameParamName =
-            org.apache.stormcrawler.opensearch.Constants.PARAMPREFIX + OSBoltType + ".index.name";
-    private static final String OSCreateParamName =
-            org.apache.stormcrawler.opensearch.Constants.PARAMPREFIX + OSBoltType + ".create";
-    private static final String OSIndexPipelineParamName =
-            org.apache.stormcrawler.opensearch.Constants.PARAMPREFIX + OSBoltType + ".pipeline";
-
-    private OutputCollector _collector;
-
-    private String indexName;
-
-    private String pipeline;
-
-    // whether the document will be created only if it does not exist or
-    // overwritten
-    private boolean create = false;
-
-    private ScopedCounter eventCounter;
-
-    private OpenSearchConnection connection;
-
-    private ScopedReducedMetric perSecMetrics;
-
-    private WaitAckCache waitAck;
-
-    public IndexerBolt() {}
-
-    /** Sets the index name instead of taking it from the configuration. * */
-    public IndexerBolt(String indexName) {
-        this.indexName = indexName;
-    }
-
-    @Override
-    public void prepare(
-            Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
-        super.prepare(conf, context, collector);
-        _collector = collector;
-        if (indexName == null) {
-            indexName = ConfUtils.getString(conf, IndexerBolt.OSIndexNameParamName, "content");
-        }
-
-        create = ConfUtils.getBoolean(conf, IndexerBolt.OSCreateParamName, false);
-        pipeline = ConfUtils.getString(conf, IndexerBolt.OSIndexPipelineParamName);
-
-        try {
-            connection = OpenSearchConnection.getConnection(conf, OSBoltType, this);
-        } catch (Exception e1) {
-            LOG.error("Can't connect to OpenSearch", e1);
-            throw new RuntimeException(e1);
-        }
-
-        this.eventCounter = CrawlerMetrics.registerCounter(context, conf, "OpensearchIndexer", 10);
-
-        this.perSecMetrics =
-                CrawlerMetrics.registerPerSecMetric(context, conf, "Indexer_average_persec", 10);
-
-        waitAck = new WaitAckCache(LOG, _collector::fail);
-        CrawlerMetrics.registerGauge(context, conf, "waitAck", waitAck::estimatedSize, 10);
-
-        // use the default status schema if none has been specified
-        try {
-            IndexCreation.checkOrCreateIndex(connection.getClient(), indexName, OSBoltType, LOG);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @Override
-    public void cleanup() {
-        waitAck.shutdown();
-        if (connection != null) {
-            connection.close();
-        }
-    }
-
-    @Override
-    public void execute(Tuple tuple) {
-
-        final String url = tuple.getStringByField("url");
-
-        // Distinguish the value used for indexing
-        // from the one used for the status
-        final String normalisedurl = valueForURL(tuple);
-
-        LOG.info("Indexing {} as {}", url, normalisedurl);
-
-        final Metadata metadata = (Metadata) tuple.getValueByField("metadata");
-
-        if (!filterDocument(metadata)) {
-            LOG.info("Filtered {}", url);
-            eventCounter.scope("Filtered").incrBy(1);
-            // treat it as successfully processed even if
-            // we do not index it
-            _collector.emit(StatusStreamName, tuple, new Values(url, metadata, Status.FETCHED));
-            _collector.ack(tuple);
-            return;
-        }
-
-        final String docID = getDocumentID(metadata, normalisedurl);
-
-        try {
-            final Map<String, Object> source = new HashMap<>();
-
-            // display text of the document?
-            if (StringUtils.isNotBlank(fieldNameForText())) {
-                final String text = trimText(tuple.getStringByField("text"));
-                if (!ignoreEmptyFields() || StringUtils.isNotBlank(text)) {
-                    source.put(fieldNameForText(), trimText(text));
-                }
-            }
-
-            // send URL as field?
-            if (StringUtils.isNotBlank(fieldNameForURL())) {
-                source.put(fieldNameForURL(), normalisedurl);
-            }
-
-            // which metadata to display?
-            final Map<String, String[]> keyVals = filterMetadata(metadata);
-
-            for (Entry<String, String[]> entry : keyVals.entrySet()) {
-                if (entry.getValue().length == 1) {
-                    final String value = entry.getValue()[0];
-                    if (!ignoreEmptyFields() || StringUtils.isNotBlank(value)) {
-                        source.put(entry.getKey(), value);
-                    }
-                } else if (entry.getValue().length > 1) {
-                    source.put(entry.getKey(), Arrays.asList(entry.getValue()));
-                }
-            }
-
-            final String targetIndex = getIndexName(metadata);
-            final BulkOperation op;
-            if (create) {
-                op =
-                        BulkOperation.of(
-                                b ->
-                                        b.create(
-                                                c -> {
-                                                    c.index(targetIndex).id(docID).document(source);
-                                                    if (pipeline != null) {
-                                                        c.pipeline(pipeline);
-                                                    }
-                                                    return c;
-                                                }));
-            } else {
-                op =
-                        BulkOperation.of(
-                                b ->
-                                        b.index(
-                                                idx -> {
-                                                    idx.index(targetIndex)
-                                                            .id(docID)
-                                                            .document(source);
-                                                    if (pipeline != null) {
-                                                        idx.pipeline(pipeline);
-                                                    }
-                                                    return idx;
-                                                }));
-            }
-
-            waitAck.addTuple(docID, tuple);
-
-            connection.addToProcessor(op);
-
-            eventCounter.scope("Indexed").incrBy(1);
-            perSecMetrics.scope("Indexed").update(1);
-        } catch (Exception e) {
-            LOG.error("Error building document for OpenSearch", e);
-            // do not send to status stream so that it gets replayed
-            _collector.fail(tuple);
-
-            waitAck.invalidate(docID);
-        }
-    }
-
-    /**
-     * Must be overridden for implementing custom index names based on some metadata information By
-     * Default, indexName coming from config is used
-     */
-    protected String getIndexName(Metadata m) {
-        return indexName;
-    }
-
-    @Override
-    public void beforeBulk(long executionId, BulkRequest request) {
-        eventCounter.scope("bulks_sent").incrBy(1);
-    }
-
-    @Override
-    public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
-        eventCounter.scope("bulks_received").incrBy(1);
-        eventCounter.scope("bulk_msec").incrBy(response.took());
-
-        waitAck.processBulkResponse(
-                response,
-                executionId,
-                eventCounter,
-                (id, t, selected) -> {
-                    String url = (String) t.getValueByField("url");
-                    Metadata metadata = (Metadata) t.getValueByField("metadata");
-
-                    if (!selected.failed()) {
-                        _collector.emit(
-                                StatusStreamName, t, new Values(url, metadata, Status.FETCHED));
-                        _collector.ack(t);
-                    } else {
-                        var failure = selected.getFailure();
-                        LOG.error("update ID {}, URL {}, failure: {}", id, url, failure);
-                        // there is something wrong with the content we should
-                        // treat it as an ERROR
-                        if (selected.getStatus() == 400) {
-                            metadata.setValue(Constants.STATUS_ERROR_SOURCE, "OpenSearch indexing");
-                            metadata.setValue(Constants.STATUS_ERROR_MESSAGE, "invalid content");
-                            _collector.emit(
-                                    StatusStreamName, t, new Values(url, metadata, Status.ERROR));
-                            _collector.ack(t);
-                            LOG.debug("Acked {} with ID {}", url, id);
-                        } else {
-                            _collector.fail(t);
-                            LOG.debug("Failed {} with ID {}", url, id);
-                        }
-                    }
-                });
-    }
-
-    @Override
-    public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
-        eventCounter.scope("bulks_received").incrBy(1);
-        waitAck.processFailedBulk(
-                request,
-                executionId,
-                failure,
-                t -> {
-                    eventCounter.scope("failed").incrBy(1);
-                    _collector.fail(t);
-                });
-    }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
deleted file mode 100644
index 49ee7f0c..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.filtering;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import java.net.URL;
-import java.util.Map;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.filtering.URLFilter;
-import org.apache.stormcrawler.opensearch.DelegateRefresher;
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-
-/**
- * Wraps a URLFilter whose resources are in a JSON file that can be stored in OpenSearch. The
- * benefit of doing this is that the resources can be refreshed automatically and modified without
- * having to recompile the jar and restart the topology. The connection to OpenSearch is done via
- * the config and uses a new bolt type 'config'.
- *
- * <p>The configuration of the delegate is done in the urlfilters.json as usual.
- *
- * <pre>
- *  {
- *     "class": "org.apache.stormcrawler.opensearch.filtering.JSONURLFilterWrapper",
- *     "name": "OSFastURLFilter",
- *     "params": {
- *         "refresh": "60",
- *         "delegate": {
- *             "class": "org.apache.stormcrawler.filtering.regex.FastURLFilter",
- *             "params": {
- *                 "file": "fast.urlfilter.json"
- *             }
- *         }
- *     }
- *  }
- * </pre>
- *
- * The resource file can be pushed to OpenSearch with
- *
- * <pre>
- *  curl -XPUT 'localhost:9200/config/config/fast.urlfilter.json?pretty' -H 'Content-Type: application/json' -d @fast.urlfilter.json
- * </pre>
- */
-public class JSONURLFilterWrapper extends URLFilter {
-
-    private DelegateRefresher<URLFilter> refresher;
-
-    public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode filterParams) {
-        refresher =
-                new DelegateRefresher<>(
-                        URLFilter.class, stormConf, filterParams, URLFilter::configure);
-    }
-
-    @Override
-    public @Nullable String filter(
-            @Nullable URL sourceUrl,
-            @Nullable Metadata sourceMetadata,
-            @NotNull String urlToFilter) {
-        return refresher.getDelegate().filter(sourceUrl, sourceMetadata, urlToFilter);
-    }
-
-    @Override
-    public void cleanup() {
-        if (refresher != null) {
-            refresher.cleanup();
-        }
-    }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
deleted file mode 100644
index c46a2b73..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.metrics;
-
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Map.Entry;
-import org.apache.storm.metric.api.IMetricsConsumer;
-import org.apache.storm.task.IErrorReporter;
-import org.apache.storm.task.TopologyContext;
-import org.apache.stormcrawler.opensearch.IndexCreation;
-import org.apache.stormcrawler.opensearch.OpenSearchConnection;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.opensearch.core.bulk.BulkOperation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Sends metrics to an OpenSearch index. The OpenSearch details are set in the configuration; an
- * optional argument sets a date format to append to the index name.
- *
- * <pre>
- *   topology.metrics.consumer.register:
- *        - class: "org.apache.stormcrawler.opensearch.metrics.MetricsConsumer"
- *          parallelism.hint: 1
- *          argument: "yyyy-MM-dd"
- * </pre>
- */
-public class MetricsConsumer implements IMetricsConsumer {
-
-    private final Logger LOG = LoggerFactory.getLogger(getClass());
-
-    private static final String OSBoltType = "metrics";
-
-    /** name of the index to use for the metrics (default : metrics) * */
-    private static final String OSMetricsIndexNameParamName =
-            "opensearch." + OSBoltType + ".index.name";
-
-    private String indexName;
-
-    private OpenSearchConnection connection;
-
-    private String stormID;
-
-    /** optional date format passed as argument, must be parsable as a SimpleDateFormat */
-    private SimpleDateFormat dateFormat;
-
-    @Override
-    public void prepare(
-            Map<String, Object> stormConf,
-            Object registrationArgument,
-            TopologyContext context,
-            IErrorReporter errorReporter) {
-        indexName = ConfUtils.getString(stormConf, OSMetricsIndexNameParamName, "metrics");
-        stormID = context.getStormId();
-        if (registrationArgument != null) {
-            dateFormat = new SimpleDateFormat((String) registrationArgument, Locale.ROOT);
-            LOG.info("Using date format {}", registrationArgument);
-        }
-        try {
-            connection = OpenSearchConnection.getConnection(stormConf, OSBoltType);
-        } catch (Exception e1) {
-            LOG.error("Can't connect to OpenSearch", e1);
-            throw new RuntimeException(e1);
-        }
-
-        // create a template if it doesn't exist
-        try {
-            IndexCreation.checkOrCreateIndexTemplate(connection.getClient(), OSBoltType, LOG);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @Override
-    public void cleanup() {
-        if (connection != null) {
-            connection.close();
-        }
-    }
-
-    @Override
-    public void handleDataPoints(TaskInfo taskInfo, Collection<DataPoint> dataPoints) {
-        final Date now = new Date();
-        for (DataPoint dataPoint : dataPoints) {
-            handleDataPoints(taskInfo, dataPoint.name, dataPoint.value, now);
-        }
-    }
-
-    private void handleDataPoints(
-            final TaskInfo taskInfo, final String nameprefix, final Object value, final Date now) {
-        if (value instanceof Number) {
-            indexDataPoint(taskInfo, now, nameprefix, ((Number) value).doubleValue());
-        } else if (value instanceof Map) {
-            for (Entry<String, Object> entry : ((Map<String, Object>) value).entrySet()) {
-                String newnameprefix = nameprefix + "." + entry.getKey();
-                handleDataPoints(taskInfo, newnameprefix, entry.getValue(), now);
-            }
-        } else if (value instanceof Collection) {
-            for (Object collectionObj : (Collection<Object>) value) {
-                handleDataPoints(taskInfo, nameprefix, collectionObj, now);
-            }
-        } else {
-            LOG.warn("Found data point value {} of {}", nameprefix, value.getClass().toString());
-        }
-    }
-
-    /**
-     * Returns the name of the index that metrics will be written to.
-     *
-     * @return elastic index name
-     */
-    private String getIndexName(Date timestamp) {
-        if (dateFormat == null) {
-            return indexName;
-        }
-
-        StringBuilder sb = new StringBuilder(indexName);
-        sb.append("-").append(dateFormat.format(timestamp));
-        return sb.toString();
-    }
-
-    private void indexDataPoint(TaskInfo taskInfo, Date timestamp, String name, double value) {
-        try {
-            Map<String, Object> doc = new HashMap<>();
-            doc.put("stormId", stormID);
-            doc.put("srcComponentId", taskInfo.srcComponentId);
-            doc.put("srcTaskId", taskInfo.srcTaskId);
-            doc.put("srcWorkerHost", taskInfo.srcWorkerHost);
-            doc.put("srcWorkerPort", taskInfo.srcWorkerPort);
-            doc.put("name", name);
-            doc.put("value", value);
-            doc.put("timestamp", timestamp.toInstant().toString());
-
-            final String targetIndex = getIndexName(timestamp);
-            BulkOperation op =
-                    BulkOperation.of(b -> b.index(idx -> idx.index(targetIndex).document(doc)));
-            connection.addToProcessor(op);
-        } catch (Exception e) {
-            LOG.error("problem when building request for OpenSearch", e);
-        }
-    }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java
deleted file mode 100644
index d3b654a0..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.metrics;
-
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.Histogram;
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.MetricFilter;
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.ScheduledReporter;
-import com.codahale.metrics.Timer;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.concurrent.TimeUnit;
-import org.apache.storm.metrics2.reporters.ScheduledStormReporter;
-import org.apache.stormcrawler.opensearch.IndexCreation;
-import org.apache.stormcrawler.opensearch.OpenSearchConnection;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.opensearch.core.bulk.BulkOperation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Storm V2 metrics reporter that writes metrics to an OpenSearch index with the same document
- * structure as the V1 {@link MetricsConsumer}. This allows existing OpenSearch dashboards to work
- * unchanged during migration from V1 to V2 metrics.
- *
- * <p>Configuration in storm.yaml:
- *
- * <pre>
- *   storm.metrics.reporters:
- *     - class: "org.apache.stormcrawler.opensearch.metrics.MetricsReporter"
- *       report.period: 10
- *       report.period.units: "SECONDS"
- * </pre>
- */
-public class MetricsReporter extends ScheduledStormReporter {
-
-    private static final Logger LOG = LoggerFactory.getLogger(MetricsReporter.class);
-
-    private static final String OSBoltType = "metrics";
-
-    private static final String OSMetricsIndexNameParamName =
-            "opensearch." + OSBoltType + ".index.name";
-
-    private static final String DATE_FORMAT_KEY = "opensearch.metrics.date.format";
-
-    private ScheduledReporter reporter;
-
-    @Override
-    public void prepare(
-            MetricRegistry metricsRegistry,
-            Map<String, Object> topoConf,
-            Map<String, Object> reporterConf) {
-
-        String indexName = ConfUtils.getString(topoConf, OSMetricsIndexNameParamName, "metrics");
-        String stormId = (String) topoConf.getOrDefault("storm.id", "unknown");
-
-        SimpleDateFormat dateFormat = null;
-        String dateFormatStr = ConfUtils.getString(topoConf, DATE_FORMAT_KEY, null);
-        if (dateFormatStr != null) {
-            dateFormat = new SimpleDateFormat(dateFormatStr, Locale.ROOT);
-        }
-
-        OpenSearchConnection connection;
-        try {
-            connection = OpenSearchConnection.getConnection(topoConf, OSBoltType);
-        } catch (Exception e) {
-            LOG.error("Can't connect to OpenSearch", e);
-            throw new RuntimeException(e);
-        }
-
-        try {
-            IndexCreation.checkOrCreateIndexTemplate(connection.getClient(), OSBoltType, LOG);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-
-        TimeUnit reportPeriodUnit = getReportPeriodUnit(reporterConf);
-        long reportPeriod = getReportPeriod(reporterConf);
-
-        reporter =
-                new OpenSearchScheduledReporter(
-                        metricsRegistry, indexName, stormId, dateFormat, connection);
-
-        reporter.start(reportPeriod, reportPeriodUnit);
-    }
-
-    @Override
-    public void start() {
-        // already started in prepare()
-    }
-
-    @Override
-    public void stop() {
-        if (reporter != null) {
-            reporter.stop();
-        }
-    }
-
-    /**
-     * Inner ScheduledReporter that writes Codahale metrics to OpenSearch in the same format as the
-     * V1 {@link MetricsConsumer}.
-     */
-    private static class OpenSearchScheduledReporter extends ScheduledReporter {
-
-        private final String indexName;
-        private final String stormId;
-        private final SimpleDateFormat dateFormat;
-        private final OpenSearchConnection connection;
-
-        OpenSearchScheduledReporter(
-                MetricRegistry registry,
-                String indexName,
-                String stormId,
-                SimpleDateFormat dateFormat,
-                OpenSearchConnection connection) {
-            super(
-                    registry,
-                    "opensearch-metrics-reporter",
-                    MetricFilter.ALL,
-                    TimeUnit.SECONDS,
-                    TimeUnit.MILLISECONDS);
-            this.indexName = indexName;
-            this.stormId = stormId;
-            this.dateFormat = dateFormat;
-            this.connection = connection;
-        }
-
-        @Override
-        @SuppressWarnings("rawtypes")
-        public void report(
-                SortedMap<String, Gauge> gauges,
-                SortedMap<String, Counter> counters,
-                SortedMap<String, Histogram> histograms,
-                SortedMap<String, Meter> meters,
-                SortedMap<String, Timer> timers) {
-
-            Date now = new Date();
-
-            for (Map.Entry<String, Gauge> entry : gauges.entrySet()) {
-                Object value = entry.getValue().getValue();
-                if (value instanceof Number) {
-                    indexDataPoint(now, entry.getKey(), ((Number) value).doubleValue());
-                } else if (value instanceof Map) {
-                    for (Map.Entry<?, ?> mapEntry : ((Map<?, ?>) value).entrySet()) {
-                        if (mapEntry.getValue() instanceof Number) {
-                            indexDataPoint(
-                                    now,
-                                    entry.getKey() + "." + mapEntry.getKey(),
-                                    ((Number) mapEntry.getValue()).doubleValue());
-                        }
-                    }
-                }
-            }
-
-            for (Map.Entry<String, Counter> entry : counters.entrySet()) {
-                indexDataPoint(now, entry.getKey(), entry.getValue().getCount());
-            }
-
-            for (Map.Entry<String, Histogram> entry : histograms.entrySet()) {
-                indexDataPoint(now, entry.getKey(), entry.getValue().getSnapshot().getMean());
-            }
-
-            for (Map.Entry<String, Meter> entry : meters.entrySet()) {
-                indexDataPoint(now, entry.getKey(), entry.getValue().getOneMinuteRate());
-            }
-
-            for (Map.Entry<String, Timer> entry : timers.entrySet()) {
-                indexDataPoint(now, entry.getKey(), entry.getValue().getSnapshot().getMean());
-            }
-        }
-
-        private String getIndexName(Date timestamp) {
-            if (dateFormat == null) {
-                return indexName;
-            }
-            return indexName + "-" + dateFormat.format(timestamp);
-        }
-
-        private void indexDataPoint(Date timestamp, String name, double value) {
-            try {
-                Map<String, Object> doc = new HashMap<>();
-                doc.put("stormId", stormId);
-                doc.put("name", name);
-                doc.put("value", value);
-                doc.put("timestamp", timestamp.toInstant().toString());
-
-                final String targetIndex = getIndexName(timestamp);
-                BulkOperation op =
-                        BulkOperation.of(b -> b.index(idx -> idx.index(targetIndex).document(doc)));
-                connection.addToProcessor(op);
-            } catch (Exception e) {
-                LOG.error("Problem when building request for OpenSearch", e);
-            }
-        }
-    }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
deleted file mode 100644
index c74184c2..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.metrics;
-
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.storm.Config;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.topology.OutputFieldsDeclarer;
-import org.apache.storm.topology.base.BaseRichBolt;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.utils.TupleUtils;
-import org.apache.stormcrawler.metrics.CrawlerMetrics;
-import org.apache.stormcrawler.opensearch.Constants;
-import org.apache.stormcrawler.opensearch.OpenSearchConnection;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.opensearch.OpenSearchClient;
-import org.opensearch.client.opensearch._types.FieldValue;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Queries the status index periodically to get the count of URLs per status. This bolt can be
- * connected to the output of any other bolt and will not produce anything as output.
- */
-public class StatusMetricsBolt extends BaseRichBolt {
-
-    private static final Logger LOG = LoggerFactory.getLogger(StatusMetricsBolt.class);
-
-    private static final String OSBoltType = "status";
-    private static final String OSStatusIndexNameParamName =
-            Constants.PARAMPREFIX + "status.index.name";
-
-    private String indexName;
-
-    private OpenSearchClient client;
-
-    private Map<String, Long> latestStatusCounts = new ConcurrentHashMap<>(6);
-
-    private int freqStats = 60;
-
-    private OutputCollector _collector;
-
-    private transient StatusCounter[] counters;
-
-    private static final class StatusCounter {
-        final String name;
-        final AtomicBoolean ready = new AtomicBoolean(true);
-
-        StatusCounter(String name) {
-            this.name = name;
-        }
-    }
-
-    @Override
-    public void prepare(
-            Map<String, Object> stormConf, TopologyContext context, OutputCollector collector) {
-        _collector = collector;
-        indexName = ConfUtils.getString(stormConf, OSStatusIndexNameParamName, "status");
-        try {
-            client = OpenSearchConnection.getClient(stormConf, OSBoltType);
-        } catch (Exception e1) {
-            LOG.error("Can't connect to OpenSearch", e1);
-            throw new RuntimeException(e1);
-        }
-
-        CrawlerMetrics.registerGauge(
-                context, stormConf, "status.count", () -> latestStatusCounts, freqStats);
-
-        counters = new StatusCounter[6];
-
-        counters[0] = new StatusCounter("DISCOVERED");
-        counters[1] = new StatusCounter("FETCHED");
-        counters[2] = new StatusCounter("FETCH_ERROR");
-        counters[3] = new StatusCounter("REDIRECTION");
-        counters[4] = new StatusCounter("ERROR");
-        counters[5] = new StatusCounter("TOTAL");
-    }
-
-    @Override
-    public Map<String, Object> getComponentConfiguration() {
-        Config conf = new Config();
-        conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, freqStats);
-        return conf;
-    }
-
-    @Override
-    public void execute(Tuple input) {
-        _collector.ack(input);
-
-        // this bolt can be connected to anything
-        // we just want to trigger a new search when the input is a tick tuple
-        if (!TupleUtils.isTick(input)) {
-            return;
-        }
-
-        for (StatusCounter counter : counters) {
-            // still waiting for results from previous request
-            if (!counter.ready.compareAndSet(true, false)) {
-                LOG.debug("Not ready to get counts for status {}", counter.name);
-                continue;
-            }
-            final String statusName = counter.name;
-            CompletableFuture.supplyAsync(
-                            () -> {
-                                try {
-                                    if (statusName.equalsIgnoreCase("TOTAL")) {
-                                        return client.count(c -> c.index(indexName));
-                                    } else {
-                                        return client.count(
-                                                c ->
-                                                        c.index(indexName)
-                                                                .query(
-                                                                        q ->
-                                                                                q.term(
-                                                                                        t ->
-                                                                                                t.field(
-                                                                                                                "status")
-                                                                                                        .value(
-                                                                                                                FieldValue
-                                                                                                                        .of(
-                                                                                                                                statusName)))));
-                                    }
-                                } catch (Exception e) {
-                                    throw new CompletionException(e);
-                                }
-                            })
-                    .thenAccept(
-                            response -> {
-                                counter.ready.set(true);
-                                LOG.debug(
-                                        "Got {} counts for status:{}",
-                                        response.count(),
-                                        statusName);
-                                latestStatusCounts.put(statusName, response.count());
-                            })
-                    .exceptionally(
-                            e -> {
-                                counter.ready.set(true);
-                                Throwable cause =
-                                        e instanceof CompletionException ? e.getCause() : e;
-                                LOG.error(
-                                        "Failure when getting counts for status:{}",
-                                        statusName,
-                                        cause);
-                                return null;
-                            });
-        }
-    }
-
-    @Override
-    public void cleanup() {
-        if (client != null) {
-            try {
-                client._transport().close();
-            } catch (Exception e) {
-                LOG.error("Exception closing client transport", e);
-            }
-        }
-    }
-
-    @Override
-    public void declareOutputFields(OutputFieldsDeclarer declarer) {
-        // NONE - THIS BOLT DOES NOT GET CONNECTED TO ANY OTHERS
-    }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
deleted file mode 100644
index 6f6d7be6..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.parse.filter;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import java.util.Map;
-import org.apache.stormcrawler.opensearch.DelegateRefresher;
-import org.apache.stormcrawler.parse.ParseFilter;
-import org.apache.stormcrawler.parse.ParseResult;
-import org.jetbrains.annotations.NotNull;
-import org.w3c.dom.DocumentFragment;
-
-/**
- * Wraps a ParseFilter whose resources are in a JSON file that can be stored in OpenSearch. The
- * benefit of doing this is that the resources can be refreshed automatically and modified without
- * having to recompile the jar and restart the topology. The connection to OpenSearch is done via
- * the config and uses a new bolt type 'config'.
- *
- * <p>The configuration of the delegate is done in the parsefilters.json as usual.
- *
- * <pre>
- *  {
- *     "class": "org.apache.stormcrawler.opensearch.parse.filter.JSONResourceWrapper",
- *     "name": "OpenSearchCollectionTagger",
- *     "params": {
- *         "refresh": "60",
- *         "delegate": {
- *             "class": "org.apache.stormcrawler.parse.filter.CollectionTagger",
- *             "params": {
- *                 "file": "collections.json"
- *             }
- *         }
- *     }
- *  }
- * </pre>
- *
- * The resource file can be pushed to OpenSearch with
- *
- * <pre>
- *  curl -XPUT "$OSHOST/config/_create/collections.json" -H 'Content-Type: application/json' -d @src/main/resources/collections.json
- * </pre>
- */
-public class JSONResourceWrapper extends ParseFilter {
-
-    private DelegateRefresher<ParseFilter> refresher;
-
-    public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode filterParams) {
-        refresher =
-                new DelegateRefresher<>(
-                        ParseFilter.class, stormConf, filterParams, ParseFilter::configure);
-    }
-
-    @Override
-    public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) {
-        refresher.getDelegate().filter(url, content, doc, parse);
-    }
-
-    @Override
-    public void cleanup() {
-        if (refresher != null) {
-            refresher.cleanup();
-        }
-    }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
deleted file mode 100644
index 0afe2b19..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
+++ /dev/null
@@ -1,330 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.persistence;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import org.apache.storm.spout.SpoutOutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.opensearch.Constants;
-import org.apache.stormcrawler.opensearch.IndexCreation;
-import org.apache.stormcrawler.opensearch.OpenSearchConnection;
-import org.apache.stormcrawler.persistence.AbstractQueryingSpout;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.json.JsonData;
-import org.opensearch.client.opensearch.OpenSearchClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class AbstractSpout extends AbstractQueryingSpout {
-
-    private static final Logger LOG = LoggerFactory.getLogger(AbstractSpout.class);
-
-    protected static final String OSBoltType = "status";
-    protected static final String OSStatusIndexNameParamName =
-            Constants.PARAMPREFIX + OSBoltType + ".index.name";
-
-    /** Field name to use for aggregating * */
-    protected static final String OSStatusBucketFieldParamName =
-            Constants.PARAMPREFIX + OSBoltType + ".bucket.field";
-
-    protected static final String OSStatusMaxBucketParamName =
-            Constants.PARAMPREFIX + OSBoltType + ".max.buckets";
-    protected static final String OSStatusMaxURLsParamName =
-            Constants.PARAMPREFIX + OSBoltType + ".max.urls.per.bucket";
-
-    /** Field name to use for sorting the URLs within a bucket, not used if empty or null. */
-    protected static final String OSStatusBucketSortFieldParamName =
-            Constants.PARAMPREFIX + OSBoltType + ".bucket.sort.field";
-
-    /** Field name to use for sorting the buckets, not used if empty or null. */
-    protected static final String OSStatusGlobalSortFieldParamName =
-            Constants.PARAMPREFIX + OSBoltType + ".global.sort.field";
-
-    protected static final String OSStatusFilterParamName =
-            Constants.PARAMPREFIX + OSBoltType + ".filterQuery";
-
-    protected static final String OSStatusQueryTimeoutParamName =
-            Constants.PARAMPREFIX + OSBoltType + ".query.timeout";
-
-    /** Query to use as a positive filter, set by es.status.filterQuery */
-    protected List<String> filterQueries = null;
-
-    protected String indexName;
-
-    protected static OpenSearchClient client;
-
-    /**
-     * when using multiple instances - each one is in charge of a specific shard useful when
-     * sharding based on host or domain to guarantee a good mix of URLs
-     */
-    protected int shardID = -1;
-
-    /** Used to distinguish between instances in the logs * */
-    protected String logIdprefix = "";
-
-    /** Field name used for field collapsing e.g. key * */
-    protected String partitionField;
-
-    protected int maxURLsPerBucket = 10;
-
-    protected int maxBucketNum = 10;
-
-    protected List<String> bucketSortField = new ArrayList<>();
-
-    protected String totalSortField = "";
-
-    protected Date queryDate;
-
-    protected int queryTimeout = -1;
-
-    /**
-     * Single-threaded executor dedicated to running the asynchronous OpenSearch queries triggered
-     * from this spout. Using a dedicated, named, daemon executor avoids polluting the JVM-wide
-     * {@link java.util.concurrent.ForkJoinPool#commonPool() common pool} (which is shared with all
-     * other Storm tasks running in the same worker) and allows clean shutdown when the topology is
-     * stopped.
-     */
-    protected ExecutorService queryExecutor;
-
-    @Override
-    public void open(
-            Map<String, Object> stormConf,
-            TopologyContext context,
-            SpoutOutputCollector collector) {
-
-        super.open(stormConf, context, collector);
-
-        final String executorName =
-                "OpenSearchSpout-"
-                        + context.getThisComponentId()
-                        + "-"
-                        + context.getThisTaskIndex();
-        queryExecutor =
-                Executors.newSingleThreadExecutor(
-                        r -> {
-                            Thread t = new Thread(r, executorName);
-                            t.setDaemon(true);
-                            return t;
-                        });
-
-        indexName = ConfUtils.getString(stormConf, OSStatusIndexNameParamName, "status");
-
-        // one OS client per JVM
-        synchronized (AbstractSpout.class) {
-            try {
-                if (client == null) {
-                    client = OpenSearchConnection.getClient(stormConf, OSBoltType);
-                }
-            } catch (Exception e1) {
-                LOG.error("Can't connect to OpenSearch", e1);
-                throw new RuntimeException(e1);
-            }
-
-            // use the default status schema if none has been specified
-            try {
-                IndexCreation.checkOrCreateIndex(client, indexName, OSBoltType, LOG);
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        }
-
-        // if more than one instance is used we expect their number to be the
-        // same as the number of shards
-        int totalTasks = context.getComponentTasks(context.getThisComponentId()).size();
-        if (totalTasks > 1) {
-            logIdprefix =
-                    "[" + context.getThisComponentId() + " #" + context.getThisTaskIndex() + "] ";
-
-            // determine the number of shards so that we can restrict the
-            // search
-
-            // TODO use the admin API when it gets available
-            // TODO or the low level one with
-            // https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-shards-stores.html
-            // TODO identify local shards and use those if possible
-
-            // ClusterSearchShardsRequest request = new
-            // ClusterSearchShardsRequest(
-            // indexName);
-            // ClusterSearchShardsResponse shardresponse = client.admin()
-            // .cluster().searchShards(request).actionGet();
-            // ClusterSearchShardsGroup[] shardgroups =
-            // shardresponse.getGroups();
-            // if (totalTasks != shardgroups.length) {
-            // throw new RuntimeException(
-            // "Number of OS spout instances should be the same as number of
-            // shards ("
-            // + shardgroups.length + ") but is " + totalTasks);
-            // }
-            // shardID = shardgroups[context.getThisTaskIndex()].getShardId()
-            // .getId();
-
-            // TEMPORARY simply use the task index as shard index
-            shardID = context.getThisTaskIndex();
-            LOG.info("{} assigned shard ID {}", logIdprefix, shardID);
-        }
-
-        partitionField = ConfUtils.getString(stormConf, OSStatusBucketFieldParamName, "key");
-
-        bucketSortField = ConfUtils.loadListFromConf(OSStatusBucketSortFieldParamName, stormConf);
-
-        totalSortField = ConfUtils.getString(stormConf, OSStatusGlobalSortFieldParamName);
-
-        maxURLsPerBucket = ConfUtils.getInt(stormConf, OSStatusMaxURLsParamName, 1);
-        maxBucketNum = ConfUtils.getInt(stormConf, OSStatusMaxBucketParamName, 10);
-
-        queryTimeout = ConfUtils.getInt(stormConf, OSStatusQueryTimeoutParamName, -1);
-
-        filterQueries = ConfUtils.loadListFromConf(OSStatusFilterParamName, stormConf);
-    }
-
-    /** Builds a query and use it retrieve the results from OS * */
-    protected abstract void populateBuffer();
-
-    /**
-     * Converts the {@link JsonData} carried by a search hit into a plain {@code Map<String,
-     * Object>} populated with native Java types (strings, numbers, lists, nested maps).
-     *
-     * <p>This helper exists to work around a subtle short-circuit inside {@code JsonDataImpl#to}:
-     *
-     * <pre>{@code
-     * if (clazz.isAssignableFrom(value.getClass())) {
-     *     return (T) value;
-     * }
-     * }</pre>
-     *
-     * <p>When {@link JsonData} is produced via {@code JsonData._DESERIALIZER} (as opposed to a raw
-     * Jackson {@code TreeNode}), its internal {@code value} is a Jakarta JSON-P {@code JsonObject}
-     * — typically a Parsson {@code JsonObjectImpl}. Crucially, {@code jakarta.json.JsonObject
-     * extends Map<String, JsonValue>}, so calling {@code source.to(Map.class)} or {@code
-     * source.to(Object.class)} matches the {@code isAssignableFrom} guard and returns the raw
-     * {@code JsonObject} <em>without ever consulting the configured {@link
-     * org.opensearch.client.json.jackson.JacksonJsonpMapper}</em>. The resulting map values are
-     * still {@code JsonString}/{@code JsonStringImpl} instances and any downstream {@code (String)
-     * value} cast blows up with a {@link ClassCastException}.
-     *
-     * <p>Passing a concrete subclass that {@code JsonObject} does <em>not</em> implement (here
-     * {@link LinkedHashMap}) bypasses the short-circuit and forces the round-trip through {@code
-     * JacksonJsonpGenerator -> JacksonJsonpParser -> Jackson ObjectMapper.readValue}, which yields
-     * native Java values (String, Long, Double, ArrayList, LinkedHashMap…).
-     *
-     * @param source the document source carried by an OpenSearch hit; may be {@code null}
-     * @return a (possibly empty) map of native Java values, never {@code null}
-     */
-    static Map<String, Object> sourceAsMap(JsonData source) {
-        if (source == null) {
-            return Collections.emptyMap();
-        }
-        @SuppressWarnings("unchecked")
-        Map<String, Object> map = source.to(LinkedHashMap.class);
-        return map == null ? Collections.emptyMap() : map;
-    }
-
-    /**
-     * Adds a document source to the URL buffer unless it is already being processed.
-     *
-     * @param source the document source as a key-value map (must contain a "url" entry)
-     * @return {@code true} if the URL was added to the buffer, {@code false} if it was already
-     *     being processed or already present
-     */
-    protected final boolean addHitToBuffer(Map<String, Object> source) {
-        String url = (String) source.get("url");
-        // is already being processed - skip it!
-        if (beingProcessed.containsKey(url)) {
-            return false;
-        }
-        return buffer.add(url, fromKeyValues(source));
-    }
-
-    protected final Metadata fromKeyValues(Map<String, Object> keyValues) {
-        Map<String, List<String>> mdAsMap = (Map<String, List<String>>) keyValues.get("metadata");
-        Metadata metadata = new Metadata();
-        if (mdAsMap != null) {
-            for (Entry<String, List<String>> mdEntry : mdAsMap.entrySet()) {
-                String key = mdEntry.getKey();
-                // periods are not allowed - replace with %2E
-                key = key.replaceAll("%2E", "\\.");
-                Object mdValObj = mdEntry.getValue();
-                // single value
-                if (mdValObj instanceof String) {
-                    metadata.addValue(key, (String) mdValObj);
-                } else {
-                    // multi valued
-                    metadata.addValues(key, (List<String>) mdValObj);
-                }
-            }
-        }
-        return metadata;
-    }
-
-    @Override
-    public void ack(Object msgId) {
-        LOG.debug("{}  Ack for {}", logIdprefix, msgId);
-        super.ack(msgId);
-    }
-
-    @Override
-    public void fail(Object msgId) {
-        LOG.info("{}  Fail for {}", logIdprefix, msgId);
-        super.fail(msgId);
-    }
-
-    @Override
-    public void close() {
-        // Shut the dedicated query executor down first so no more requests can be enqueued
-        // against the client while we are tearing it down. We give in-flight queries a short
-        // grace period to drain to keep shutdown reasonably graceful.
-        if (queryExecutor != null) {
-            queryExecutor.shutdown();
-            try {
-                if (!queryExecutor.awaitTermination(30, TimeUnit.SECONDS)) {
-                    LOG.warn(
-                            "{} Query executor did not terminate within 30s, forcing shutdown",
-                            logIdprefix);
-                    queryExecutor.shutdownNow();
-                }
-            } catch (InterruptedException e) {
-                queryExecutor.shutdownNow();
-                Thread.currentThread().interrupt();
-            }
-            queryExecutor = null;
-        }
-
-        synchronized (AbstractSpout.class) {
-            if (client != null) {
-                try {
-                    client._transport().close();
-                } catch (IOException e) {
-                    LOG.error("Exception caught when closing client", e);
-                }
-                client = null;
-            }
-        }
-    }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java
deleted file mode 100644
index a6272f36..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java
+++ /dev/null
@@ -1,432 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.persistence;
-
-import java.io.IOException;
-import java.time.Instant;
-import java.time.ZoneOffset;
-import java.time.format.DateTimeFormatter;
-import java.util.Calendar;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.storm.spout.SpoutOutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.opensearch.Constants;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.json.JsonData;
-import org.opensearch.client.opensearch._types.FieldValue;
-import org.opensearch.client.opensearch._types.SortOrder;
-import org.opensearch.client.opensearch._types.aggregations.Aggregate;
-import org.opensearch.client.opensearch._types.aggregations.Aggregation;
-import org.opensearch.client.opensearch._types.aggregations.StringTermsBucket;
-import org.opensearch.client.opensearch._types.aggregations.TopHitsAggregate;
-import org.opensearch.client.opensearch.core.SearchRequest;
-import org.opensearch.client.opensearch.core.SearchResponse;
-import org.opensearch.client.opensearch.core.search.Hit;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Spout which pulls URL from an OpenSearch index. Use a single instance unless you use
- * 'opensearch.status.routing' with the StatusUpdaterBolt, in which case you need to have exactly
- * the same number of spout instances as OpenSearch shards. Guarantees a good mix of URLs by
- * aggregating them by an arbitrary field e.g. key.
- */
-public class AggregationSpout extends AbstractSpout {
-
-    private static final Logger LOG = LoggerFactory.getLogger(AggregationSpout.class);
-
-    private static final String StatusSampleParamName = Constants.PARAMPREFIX + "status.sample";
-    private static final String MostRecentDateIncreaseParamName =
-            Constants.PARAMPREFIX + "status.recentDate.increase";
-    private static final String MostRecentDateMinGapParamName =
-            Constants.PARAMPREFIX + "status.recentDate.min.gap";
-
-    private boolean sample = false;
-
-    private int recentDateIncrease = -1;
-    private int recentDateMinGap = -1;
-
-    protected Set<String> currentBuckets;
-
-    @Override
-    public void open(
-            Map<String, Object> stormConf,
-            TopologyContext context,
-            SpoutOutputCollector collector) {
-        sample = ConfUtils.getBoolean(stormConf, StatusSampleParamName, sample);
-        recentDateIncrease =
-                ConfUtils.getInt(stormConf, MostRecentDateIncreaseParamName, recentDateIncrease);
-        recentDateMinGap =
-                ConfUtils.getInt(stormConf, MostRecentDateMinGapParamName, recentDateMinGap);
-        super.open(stormConf, context, collector);
-        currentBuckets = new HashSet<>();
-    }
-
-    @Override
-    protected void populateBuffer() {
-
-        if (queryDate == null) {
-            queryDate = new Date();
-            lastTimeResetToNow = Instant.now();
-        }
-
-        String formattedQueryDate =
-                Instant.ofEpochMilli(queryDate.getTime())
-                        .atOffset(ZoneOffset.UTC)
-                        .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
-
-        LOG.info("{} Populating buffer with nextFetchDate <= {}", logIdprefix, formattedQueryDate);
-
-        // Build the top_hits sub-aggregation
-        Aggregation topHitsAgg =
-                Aggregation.of(
-                        a ->
-                                a.topHits(
-                                        th -> {
-                                            th.size(maxURLsPerBucket).explain(false);
-                                            for (String bsf : bucketSortField) {
-                                                th.sort(
-                                                        s ->
-                                                                s.field(
-                                                                        fs ->
-                                                                                fs.field(bsf)
-                                                                                        .order(
-                                                                                                SortOrder
-                                                                                                        .Asc)));
-                                            }
-                                            return th;
-                                        }));
-
-        // Build the terms (partition) aggregation with top_hits sub-agg
-        Aggregation.Builder.ContainerBuilder partitionAggBuilder =
-                new Aggregation.Builder()
-                        .terms(
-                                t -> {
-                                    t.field(partitionField).size(maxBucketNum);
-                                    // sort between buckets by the min sub-aggregation
-                                    if (StringUtils.isNotBlank(totalSortField)) {
-                                        t.order(
-                                                Collections.singletonList(
-                                                        Collections.singletonMap(
-                                                                "top_hit", SortOrder.Asc)));
-                                    }
-                                    return t;
-                                })
-                        .aggregations("docs", topHitsAgg);
-
-        // add the min sub-aggregation used for sorting between buckets
-        if (StringUtils.isNotBlank(totalSortField)) {
-            partitionAggBuilder.aggregations(
-                    "top_hit", Aggregation.of(minAgg -> minAgg.min(m -> m.field(totalSortField))));
-        }
-
-        Aggregation partitionAgg = partitionAggBuilder.build();
-
-        // Build the search request
-        SearchRequest.Builder requestBuilder =
-                new SearchRequest.Builder()
-                        .index(indexName)
-                        .size(0)
-                        .trackTotalHits(t -> t.enabled(false))
-                        .query(
-                                q ->
-                                        q.bool(
-                                                b -> {
-                                                    b.filter(
-                                                            f ->
-                                                                    f.range(
-                                                                            r ->
-                                                                                    r.field(
-                                                                                                    "nextFetchDate")
-                                                                                            .lte(
-                                                                                                    JsonData
-                                                                                                            .of(
-                                                                                                                    formattedQueryDate))));
-                                                    if (filterQueries != null) {
-                                                        for (String fq : filterQueries) {
-                                                            b.filter(
-                                                                    f ->
-                                                                            f.queryString(
-                                                                                    qs ->
-                                                                                            qs
-                                                                                                    .query(
-                                                                                                            fq)));
-                                                        }
-                                                    }
-                                                    return b;
-                                                }));
-
-        if (queryTimeout != -1) {
-            requestBuilder.timeout(queryTimeout + "s");
-        }
-
-        if (sample) {
-            // Wrap in a diversified sampler aggregation
-            requestBuilder.aggregations(
-                    "sample",
-                    Aggregation.of(
-                            a ->
-                                    a.diversifiedSampler(
-                                                    ds ->
-                                                            ds.field(partitionField)
-                                                                    .maxDocsPerValue(
-                                                                            maxURLsPerBucket)
-                                                                    .shardSize(
-                                                                            maxURLsPerBucket
-                                                                                    * maxBucketNum))
-                                            .aggregations("partition", partitionAgg)));
-        } else {
-            requestBuilder.aggregations("partition", partitionAgg);
-        }
-
-        // shard preference for routing
-        if (shardID != -1) {
-            requestBuilder.preference("_shards:" + shardID + "|_local");
-        }
-
-        SearchRequest request = requestBuilder.build();
-
-        // dump query to log
-        LOG.debug("{} OpenSearch query {}", logIdprefix, request);
-
-        LOG.trace("{} isInQuery set to true", logIdprefix);
-        isInQuery.set(true);
-
-        CompletableFuture.supplyAsync(
-                        () -> {
-                            try {
-                                return client.search(request, JsonData.class);
-                            } catch (IOException e) {
-                                throw new CompletionException(e);
-                            }
-                        },
-                        queryExecutor)
-                .thenAccept(this::handleResponse)
-                .exceptionally(
-                        e -> {
-                            Throwable cause = e instanceof CompletionException ? e.getCause() : e;
-                            LOG.error("{} Exception with OpenSearch query", logIdprefix, cause);
-                            markQueryReceivedNow();
-                            return null;
-                        });
-    }
-
-    /**
-     * Handles the search response from an asynchronous aggregation query, extracting URLs from term
-     * buckets and adding them to the buffer.
-     *
-     * @param response the search response containing aggregation results
-     */
-    protected void handleResponse(SearchResponse<JsonData> response) {
-        long timeTaken = System.currentTimeMillis() - getTimeLastQuerySent();
-
-        Map<String, Aggregate> aggregs = response.aggregations();
-
-        if (aggregs == null || aggregs.isEmpty()) {
-            markQueryReceivedNow();
-            return;
-        }
-
-        // Unwrap the sample aggregation if present
-        Aggregate sampleAgg = aggregs.get("sample");
-        if (sampleAgg != null) {
-            aggregs = sampleAgg.sampler().aggregations();
-        }
-
-        Aggregate partitionAgg = aggregs.get("partition");
-        List<StringTermsBucket> buckets = partitionAgg.sterms().buckets().array();
-
-        int numhits = 0;
-        int numBuckets = 0;
-        int alreadyprocessed = 0;
-
-        Instant mostRecentDateFound = null;
-
-        currentBuckets.clear();
-
-        // For each entry
-        Iterator<StringTermsBucket> iterator = buckets.iterator();
-        while (iterator.hasNext()) {
-            StringTermsBucket entry = iterator.next();
-            String key = entry.key(); // bucket key
-
-            currentBuckets.add(key);
-
-            long docCount = entry.docCount(); // Doc count
-
-            int hitsForThisBucket = 0;
-
-            List<FieldValue> lastSortValues = null;
-            // filter results so that we don't include URLs we are already
-            // being processed
-            TopHitsAggregate topHits = entry.aggregations().get("docs").topHits();
-            for (Hit<JsonData> hit : topHits.hits().hits()) {
-
-                Map<String, Object> keyValues = sourceAsMap(hit.source());
-
-                LOG.debug("{} -> id [{}], _source [{}]", logIdprefix, hit.id(), keyValues);
-
-                hitsForThisBucket++;
-
-                lastSortValues = hit.sort();
-
-                String url = (String) keyValues.get("url");
-
-                // consider only the first document of the last bucket
-                // for optimising the nextFetchDate
-                if (hitsForThisBucket == 1 && !iterator.hasNext()) {
-                    String strDate = (String) keyValues.get("nextFetchDate");
-                    try {
-                        mostRecentDateFound = Instant.parse(strDate);
-                    } catch (Exception e) {
-                        throw new RuntimeException("can't parse date :" + strDate);
-                    }
-                }
-
-                // is already being processed or in buffer - skip it!
-                if (beingProcessed.containsKey(url)) {
-                    LOG.debug("{} -> already processed: {}", logIdprefix, url);
-                    alreadyprocessed++;
-                    continue;
-                }
-
-                Metadata metadata = fromKeyValues(keyValues);
-                boolean added = buffer.add(url, metadata);
-                if (!added) {
-                    LOG.debug("{} -> already in buffer: {}", logIdprefix, url);
-                    alreadyprocessed++;
-                    continue;
-                }
-                LOG.debug("{} -> added to buffer : {}", logIdprefix, url);
-            }
-
-            if (lastSortValues != null && !lastSortValues.isEmpty()) {
-                sortValuesForKey(key, lastSortValues.toArray());
-            }
-
-            if (hitsForThisBucket > 0) {
-                numBuckets++;
-            }
-
-            numhits += hitsForThisBucket;
-
-            LOG.debug(
-                    "{} key [{}], hits[{}], doc_count [{}], already_processed [{}]",
-                    logIdprefix,
-                    key,
-                    hitsForThisBucket,
-                    docCount,
-                    alreadyprocessed);
-        }
-
-        LOG.info(
-                "{} OpenSearch query returned {} hits from {} buckets in {} msec with {} already being processed. Took {} msec per doc on average.",
-                logIdprefix,
-                numhits,
-                numBuckets,
-                timeTaken,
-                alreadyprocessed,
-                ((float) timeTaken / numhits));
-
-        queryTimes.accept(timeTaken);
-        eventCounter.scope("already_being_processed").incrBy(alreadyprocessed);
-        eventCounter.scope("OpenSearch_queries").incrBy(1);
-        eventCounter.scope("OpenSearch_docs").incrBy(numhits);
-
-        // optimise the nextFetchDate by getting the most recent value
-        // returned in the query and add to it, unless the previous value is
-        // within n mins in which case we'll keep it
-        if (mostRecentDateFound != null && recentDateIncrease >= 0) {
-            Calendar potentialNewDate =
-                    Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
-            potentialNewDate.setTimeInMillis(mostRecentDateFound.toEpochMilli());
-            potentialNewDate.add(Calendar.MINUTE, recentDateIncrease);
-            Date oldDate = null;
-            // check boundaries
-            if (this.recentDateMinGap > 0) {
-                Calendar low = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
-                low.setTime(queryDate);
-                low.add(Calendar.MINUTE, -recentDateMinGap);
-                Calendar high = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
-                high.setTime(queryDate);
-                high.add(Calendar.MINUTE, recentDateMinGap);
-                if (high.before(potentialNewDate) || low.after(potentialNewDate)) {
-                    oldDate = queryDate;
-                }
-            } else {
-                oldDate = queryDate;
-            }
-            if (oldDate != null) {
-                queryDate = potentialNewDate.getTime();
-                LOG.info(
-                        "{} queryDate changed from {} to {} based on mostRecentDateFound {}",
-                        logIdprefix,
-                        oldDate,
-                        queryDate,
-                        mostRecentDateFound);
-            } else {
-                LOG.info(
-                        "{} queryDate kept at {} based on mostRecentDateFound {}",
-                        logIdprefix,
-                        queryDate,
-                        mostRecentDateFound);
-            }
-        }
-
-        // reset the value for next fetch date if the previous one is too old
-        if (resetFetchDateAfterNSecs != -1) {
-            Instant changeNeededOn =
-                    Instant.ofEpochMilli(
-                            lastTimeResetToNow.toEpochMilli() + (resetFetchDateAfterNSecs * 1000L));
-            if (Instant.now().isAfter(changeNeededOn)) {
-                LOG.info(
-                        "{} queryDate set to null based on resetFetchDateAfterNSecs {}",
-                        logIdprefix,
-                        resetFetchDateAfterNSecs);
-                queryDate = null;
-            }
-        }
-
-        // change the date if we don't get any results at all
-        if (numBuckets == 0) {
-            queryDate = null;
-        }
-
-        // remove lock
-        markQueryReceivedNow();
-    }
-
-    /**
-     * Receives the sort values of the last hit returned for a bucket, as the {@link FieldValue}
-     * instances of the client. Subclasses can pass them to {@code search_after} to read the bucket
-     * from that position.
-     */
-    protected void sortValuesForKey(String key, Object[] sortValues) {}
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
deleted file mode 100644
index b3301e5c..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.persistence;
-
-import com.github.benmanes.caffeine.cache.Cache;
-import com.github.benmanes.caffeine.cache.Caffeine;
-import java.io.IOException;
-import java.time.Instant;
-import java.time.ZoneOffset;
-import java.time.format.DateTimeFormatter;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
-import org.apache.storm.spout.SpoutOutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.stormcrawler.opensearch.Constants;
-import org.apache.stormcrawler.persistence.EmptyQueueListener;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.json.JsonData;
-import org.opensearch.client.opensearch._types.FieldValue;
-import org.opensearch.client.opensearch._types.SortOrder;
-import org.opensearch.client.opensearch.core.SearchRequest;
-import org.opensearch.client.opensearch.core.SearchResponse;
-import org.opensearch.client.opensearch.core.search.Hit;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Uses collapsing spouts to get an initial set of URLs and keys to query for and gets emptyQueue
- * notifications from the URLBuffer to query OpenSearch for a specific key.
- *
- * @since 1.15
- */
-public class HybridSpout extends AggregationSpout implements EmptyQueueListener {
-
-    private static final Logger LOG = LoggerFactory.getLogger(HybridSpout.class);
-
-    protected static final String RELOADPARAMNAME =
-            Constants.PARAMPREFIX + "status.max.urls.per.reload";
-
-    private int bufferReloadSize = 10;
-
-    private Cache<String, Object[]> searchAfterCache;
-
-    private HostResultListener hrl;
-
-    @Override
-    public void open(
-            Map<String, Object> stormConf,
-            TopologyContext context,
-            SpoutOutputCollector collector) {
-        super.open(stormConf, context, collector);
-        bufferReloadSize = ConfUtils.getInt(stormConf, RELOADPARAMNAME, maxURLsPerBucket);
-        buffer.setEmptyQueueListener(this);
-        searchAfterCache = Caffeine.newBuilder().build();
-        hrl = new HostResultListener();
-    }
-
-    @Override
-    public void emptyQueue(String queueName) {
-
-        LOG.info("{} Emptied buffer queue for {}", logIdprefix, queueName);
-
-        if (!currentBuckets.contains(queueName)) {
-            // not interested in this one any more
-            return;
-        }
-
-        // reloading the aggregs - searching now
-        // would just overload OpenSearch and yield
-        // mainly duplicates
-        if (isInQuery.get()) {
-            LOG.trace("{} isInquery true for {}", logIdprefix, queueName);
-            return;
-        }
-
-        LOG.info("{} Querying for more docs for {}", logIdprefix, queueName);
-
-        if (queryDate == null) {
-            queryDate = new Date();
-            lastTimeResetToNow = Instant.now();
-        }
-
-        String formattedQueryDate =
-                Instant.ofEpochMilli(queryDate.getTime())
-                        .atOffset(ZoneOffset.UTC)
-                        .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
-
-        SearchRequest.Builder requestBuilder =
-                new SearchRequest.Builder()
-                        .index(indexName)
-                        .size(bufferReloadSize)
-                        .trackTotalHits(t -> t.enabled(false))
-                        .query(
-                                q ->
-                                        q.bool(
-                                                b ->
-                                                        b.filter(
-                                                                        f ->
-                                                                                f.range(
-                                                                                        r ->
-                                                                                                r.field(
-                                                                                                                "nextFetchDate")
-                                                                                                        .lte(
-                                                                                                                JsonData
-                                                                                                                        .of(
-                                                                                                                                formattedQueryDate))))
-                                                                .filter(
-                                                                        f ->
-                                                                                f.term(
-                                                                                        t ->
-                                                                                                t.field(
-                                                                                                                partitionField)
-                                                                                                        .value(
-                                                                                                                FieldValue
-                                                                                                                        .of(
-                                                                                                                                queueName))))));
-
-        // sort within a bucket
-        for (String bsf : bucketSortField) {
-            requestBuilder.sort(s -> s.field(fs -> fs.field(bsf).order(SortOrder.Asc)));
-        }
-
-        // do we have a search after for this one?
-        Object[] searchAfterValues = searchAfterCache.getIfPresent(queueName);
-        if (searchAfterValues != null) {
-            for (Object sav : searchAfterValues) {
-                requestBuilder.searchAfter((FieldValue) sav);
-            }
-        }
-
-        // shard preference for routing
-        if (shardID != -1) {
-            requestBuilder.preference("_shards:" + shardID + "|_local");
-        }
-
-        SearchRequest request = requestBuilder.build();
-
-        // dump query to log
-        LOG.debug("{} OpenSearch query {} - {}", logIdprefix, queueName, request);
-
-        CompletableFuture.supplyAsync(
-                        () -> {
-                            try {
-                                return client.search(request, JsonData.class);
-                            } catch (IOException e) {
-                                throw new CompletionException(e);
-                            }
-                        },
-                        queryExecutor)
-                .thenAccept(hrl::handleResponse)
-                .exceptionally(
-                        e -> {
-                            Throwable cause = e instanceof CompletionException ? e.getCause() : e;
-                            LOG.error("Exception with OpenSearch query", cause);
-                            return null;
-                        });
-    }
-
-    /** Overrides the handling of responses for aggregations. */
-    @Override
-    protected void handleResponse(SearchResponse<JsonData> response) {
-        // delete all entries from the searchAfterCache when
-        // we get the results from the aggregation spouts
-        searchAfterCache.invalidateAll();
-        super.handleResponse(response);
-    }
-
-    /** The aggregation kindly told us where to start from. */
-    @Override
-    protected void sortValuesForKey(String key, Object[] sortValues) {
-        if (sortValues != null && sortValues.length > 0) {
-            this.searchAfterCache.put(key, sortValues);
-        }
-    }
-
-    /** Handling of results for a specific queue. */
-    class HostResultListener {
-
-        /**
-         * Handles the search response for a host-specific query, extracting hits and adding them to
-         * the buffer.
-         *
-         * @param response the search response containing document hits
-         */
-        void handleResponse(SearchResponse<JsonData> response) {
-
-            int alreadyprocessed = 0;
-            int numDocs = 0;
-
-            List<Hit<JsonData>> hits = response.hits().hits();
-
-            Object[] sortValues = null;
-
-            // retrieve the key for these results
-            String key = null;
-
-            for (Hit<JsonData> hit : hits) {
-                numDocs++;
-
-                Map<String, Object> source = sourceAsMap(hit.source());
-
-                String pfield = partitionField;
-                Map<String, Object> fieldSource = source;
-                if (pfield.startsWith("metadata.")) {
-                    @SuppressWarnings("unchecked")
-                    Map<String, Object> metadataMap = (Map<String, Object>) source.get("metadata");
-                    fieldSource = metadataMap;
-                    pfield = pfield.substring(9);
-                }
-                Object key_as_object = fieldSource.get(pfield);
-                if (key_as_object instanceof List) {
-                    @SuppressWarnings("unchecked")
-                    List<String> keyList = (List<String>) key_as_object;
-                    if (keyList.size() == 1) {
-                        key = keyList.get(0);
-                    }
-                } else {
-                    key = key_as_object.toString();
-                }
-
-                sortValues = hit.sort().toArray();
-                if (!addHitToBuffer(source)) {
-                    alreadyprocessed++;
-                }
-            }
-
-            // no key if no results have been found
-            if (key != null) {
-                searchAfterCache.put(key, sortValues);
-            }
-
-            eventCounter.scope("OpenSearch_queries_host").incrBy(1);
-            eventCounter.scope("OpenSearch_docs_host").incrBy(numDocs);
-            eventCounter.scope("already_being_processed_host").incrBy(alreadyprocessed);
-
-            LOG.info(
-                    "{} OpenSearch term query returned {} hits  in {} msec with {} already being processed for {}",
-                    logIdprefix,
-                    numDocs,
-                    response.took(),
-                    alreadyprocessed,
-                    key);
-        }
-    }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java
deleted file mode 100644
index 6af7d909..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.persistence;
-
-import com.github.benmanes.caffeine.cache.Cache;
-import com.github.benmanes.caffeine.cache.Caffeine;
-import java.io.IOException;
-import java.time.Instant;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.topology.OutputFieldsDeclarer;
-import org.apache.storm.topology.base.BaseRichBolt;
-import org.apache.storm.tuple.Tuple;
-import org.apache.stormcrawler.opensearch.AsyncBulkProcessor;
-import org.apache.stormcrawler.opensearch.Constants;
-import org.apache.stormcrawler.opensearch.IndexCreation;
-import org.apache.stormcrawler.opensearch.OpenSearchConnection;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.opensearch.core.BulkRequest;
-import org.opensearch.client.opensearch.core.BulkResponse;
-import org.opensearch.client.opensearch.core.bulk.BulkOperation;
-import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Sends information about the queues into an OpenSearch index. This has to be connected to a status
- * updater bolt.
- */
-public class QueueBolt extends BaseRichBolt implements AsyncBulkProcessor.Listener {
-
-    private static final Logger LOG = LoggerFactory.getLogger(QueueBolt.class);
-
-    private static final String OSBoltType = "queues";
-
-    static final String OSQueuesIndexNameParamName =
-            Constants.PARAMPREFIX + OSBoltType + ".index.name";
-
-    /**
-     * Parameter name to configure the cache of known queues. @see
-     * https://github.com/ben-manes/caffeine/wiki/Specification Default value is
-     * "maximumSize=10000".
-     */
-    static final String OSQueuesCacheSpecParamName =
-            Constants.PARAMPREFIX + OSBoltType + ".cache.spec";
-
-    private OutputCollector _collector;
-
-    private String indexName;
-
-    private OpenSearchConnection connection;
-
-    /** Keys confirmed to be present in the index; populated on bulk success only */
-    private Cache<String, Boolean> knownQueue;
-
-    /** docID to key for the requests currently in the bulk processor */
-    private final Map<String, String> inFlight = new ConcurrentHashMap<>();
-
-    public QueueBolt() {}
-
-    /** Sets the index name instead of taking it from the configuration. * */
-    public QueueBolt(String indexName) {
-        this.indexName = indexName;
-    }
-
-    @Override
-    public void prepare(
-            Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
-        _collector = collector;
-        if (indexName == null) {
-            indexName = ConfUtils.getString(conf, QueueBolt.OSQueuesIndexNameParamName, OSBoltType);
-        }
-        try {
-            connection = OpenSearchConnection.getConnection(conf, OSBoltType, this);
-        } catch (Exception e1) {
-            LOG.error("Can't connect to OpenSearch", e1);
-            throw new RuntimeException(e1);
-        }
-        try {
-            IndexCreation.checkOrCreateIndex(connection.getClient(), indexName, OSBoltType, LOG);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-
-        final String cacheSpec =
-                ConfUtils.getString(
-                        conf, QueueBolt.OSQueuesCacheSpecParamName, "maximumSize=10000");
-        knownQueue = Caffeine.from(cacheSpec).build();
-    }
-
-    @Override
-    public void cleanup() {
-        if (connection != null) {
-            connection.close();
-        }
-    }
-
-    @Override
-    public void execute(Tuple tuple) {
-
-        final String key = tuple.getStringByField("key");
-
-        // ack no matter what - the queue info is best effort bookkeeping
-        // and failed writes are retried when the key is next seen
-        _collector.ack(tuple);
-
-        // check whether this key is already known
-        if (knownQueue.getIfPresent(key) != null) {
-            return;
-        }
-
-        final String docID = org.apache.commons.codec.digest.DigestUtils.sha256Hex(key);
-
-        // a request for this key is already in the bulk processor
-        if (inFlight.putIfAbsent(docID, key) != null) {
-            return;
-        }
-
-        final HashMap<String, String> fields = new HashMap<>();
-        fields.put("key", key);
-        fields.put("lastUpdated", Instant.now().toString());
-
-        final BulkOperation op =
-                BulkOperation.of(b -> b.create(c -> c.index(indexName).id(docID).document(fields)));
-
-        connection.addToProcessor(op);
-    }
-
-    @Override
-    public void beforeBulk(long executionId, BulkRequest request) {
-        LOG.debug("beforeBulk {} with {} actions", executionId, request.operations().size());
-    }
-
-    @Override
-    public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
-        LOG.debug("afterBulk [{}] with {} responses", executionId, response.items().size());
-        for (BulkResponseItem item : response.items()) {
-            final String key = inFlight.remove(item.id());
-            if (key == null) {
-                continue;
-            }
-            if (item.error() == null) {
-                knownQueue.put(key, Boolean.TRUE);
-            } else if (item.status() == 409) {
-                // entry already exists in the index, e.g. written before a worker
-                // restart or evicted from the cache - not an error
-                knownQueue.put(key, Boolean.TRUE);
-            } else {
-                // do not cache the key so that the write is retried
-                // when the key is next seen
-                final var error = item.error();
-                LOG.error(
-                        "Failed to index queue entry {}: {}",
-                        key,
-                        error.reason() != null ? error.reason() : error.type());
-            }
-        }
-    }
-
-    @Override
-    public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
-        LOG.error("Exception with bulk {} - failing the whole lot ", executionId, failure);
-        for (BulkOperation op : request.operations()) {
-            // not cached - the writes are retried when the keys are next seen
-            inFlight.remove(OpenSearchConnection.getBulkOperationId(op));
-        }
-    }
-
-    @Override
-    public void declareOutputFields(OutputFieldsDeclarer declarer) {
-        // nothing to do here - this bolt is the last of a topology
-    }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
deleted file mode 100644
index 2dcd76fe..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
+++ /dev/null
@@ -1,358 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.persistence;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.tuple.Values;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.metrics.CrawlerMetrics;
-import org.apache.stormcrawler.metrics.ScopedCounter;
-import org.apache.stormcrawler.metrics.ScopedReducedMetric;
-import org.apache.stormcrawler.opensearch.AsyncBulkProcessor;
-import org.apache.stormcrawler.opensearch.Constants;
-import org.apache.stormcrawler.opensearch.IndexCreation;
-import org.apache.stormcrawler.opensearch.OpenSearchConnection;
-import org.apache.stormcrawler.opensearch.WaitAckCache;
-import org.apache.stormcrawler.persistence.AbstractStatusUpdaterBolt;
-import org.apache.stormcrawler.persistence.Status;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.apache.stormcrawler.util.URLPartitioner;
-import org.opensearch.client.opensearch.core.BulkRequest;
-import org.opensearch.client.opensearch.core.BulkResponse;
-import org.opensearch.client.opensearch.core.bulk.BulkOperation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Simple bolt which stores the status of URLs into OpenSearch. Takes the tuples coming from the
- * 'status' stream. To be used in combination with a Spout to read from the index.
- */
-public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt
-        implements AsyncBulkProcessor.Listener {
-
-    private static final Logger LOG = LoggerFactory.getLogger(StatusUpdaterBolt.class);
-
-    private String OSBoltType = "status";
-
-    private static final String OSStatusIndexNameParamName =
-            Constants.PARAMPREFIX + "%s.index.name";
-    private static final String OSStatusRoutingParamName = Constants.PARAMPREFIX + "%s.routing";
-    private static final String OSStatusRoutingFieldParamName =
-            Constants.PARAMPREFIX + "%s.routing.fieldname";
-
-    private boolean routingFieldNameInMetadata = false;
-
-    private String indexName;
-
-    private URLPartitioner partitioner;
-
-    /** whether to apply the same partitioning logic used for politeness for routing, e.g byHost */
-    private boolean doRouting;
-
-    /** Store the key used for routing explicitly as a field in metadata * */
-    private String fieldNameForRoutingKey = null;
-
-    private OpenSearchConnection connection;
-
-    private WaitAckCache waitAck;
-
-    private ScopedCounter eventCounter;
-
-    private ScopedReducedMetric receivedPerSecMetrics;
-
-    public StatusUpdaterBolt() {
-        super();
-    }
-
-    /**
-     * Loads the configuration using a substring different from the default value 'status' in order
-     * to distinguish it from the spout configurations
-     */
-    public StatusUpdaterBolt(String boltType) {
-        super();
-        OSBoltType = boltType;
-    }
-
-    @Override
-    public void prepare(
-            Map<String, Object> stormConf, TopologyContext context, OutputCollector collector) {
-
-        super.prepare(stormConf, context, collector);
-
-        indexName =
-                ConfUtils.getString(
-                        stormConf,
-                        String.format(
-                                Locale.ROOT,
-                                StatusUpdaterBolt.OSStatusIndexNameParamName,
-                                OSBoltType),
-                        "status");
-
-        doRouting =
-                ConfUtils.getBoolean(
-                        stormConf,
-                        String.format(
-                                Locale.ROOT,
-                                StatusUpdaterBolt.OSStatusRoutingParamName,
-                                OSBoltType),
-                        false);
-
-        partitioner = new URLPartitioner();
-        partitioner.configure(stormConf);
-
-        fieldNameForRoutingKey =
-                ConfUtils.getString(
-                        stormConf,
-                        String.format(
-                                Locale.ROOT,
-                                StatusUpdaterBolt.OSStatusRoutingFieldParamName,
-                                OSBoltType));
-        if (StringUtils.isNotBlank(fieldNameForRoutingKey)) {
-            if (fieldNameForRoutingKey.startsWith("metadata.")) {
-                routingFieldNameInMetadata = true;
-                fieldNameForRoutingKey = fieldNameForRoutingKey.substring("metadata.".length());
-            }
-            // periods are not allowed in - replace with %2E
-            fieldNameForRoutingKey = fieldNameForRoutingKey.replaceAll("\\.", "%2E");
-        }
-
-        int metrics_time_bucket_secs = 30;
-
-        // benchmarking - average number of items received back from OpenSearch per second
-        this.receivedPerSecMetrics =
-                CrawlerMetrics.registerPerSecMetric(
-                        context, stormConf, "average_persec", metrics_time_bucket_secs);
-
-        // eventCounter MUST be registered before WaitAckCache — the eviction lambda captures it
-        this.eventCounter =
-                CrawlerMetrics.registerCounter(
-                        context, stormConf, "counters", metrics_time_bucket_secs);
-
-        String defaultSpec =
-                String.format(
-                        Locale.ROOT,
-                        "expireAfterWrite=%ds",
-                        ConfUtils.getInt(stormConf, "topology.message.timeout.secs", 300));
-
-        String waitAckSpec =
-                ConfUtils.getString(stormConf, "opensearch.status.waitack.cache.spec", defaultSpec);
-
-        waitAck =
-                new WaitAckCache(
-                        waitAckSpec,
-                        LOG,
-                        t -> {
-                            eventCounter.scope("purged").incrBy(1);
-                            collector.fail(t);
-                        });
-        CrawlerMetrics.registerGauge(
-                context, stormConf, "waitAck", waitAck::estimatedSize, metrics_time_bucket_secs);
-
-        try {
-            connection = OpenSearchConnection.getConnection(stormConf, OSBoltType, this);
-        } catch (Exception e1) {
-            LOG.error("Can't connect to OpenSearch", e1);
-            throw new RuntimeException(e1);
-        }
-
-        // use the default status schema if none has been specified
-        try {
-            IndexCreation.checkOrCreateIndex(connection.getClient(), indexName, OSBoltType, LOG);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @Override
-    public void cleanup() {
-        waitAck.shutdown();
-        if (connection == null) {
-            return;
-        }
-        connection.close();
-        connection = null;
-    }
-
-    @Override
-    public void store(
-            String url, Status status, Metadata metadata, Optional<Date> nextFetch, Tuple tuple)
-            throws Exception {
-
-        String documentID = getDocumentID(metadata, url);
-
-        boolean isAlreadySentAndDiscovered =
-                status.equals(Status.DISCOVERED) && waitAck.contains(documentID);
-
-        if (isAlreadySentAndDiscovered) {
-            // if this object is discovered - adding another version of it
-            // won't make any difference
-            LOG.debug(
-                    "Already being sent to OpenSearch {} with status {} and ID {}",
-                    url,
-                    status,
-                    documentID);
-            // ack straight away!
-            eventCounter.scope("skipped").incrBy(1);
-            super.ack(tuple, url);
-            return;
-        }
-
-        Map<String, Object> doc = new HashMap<>();
-        doc.put("url", url);
-        doc.put("status", status.name());
-
-        Map<String, Object> metadataMap = new HashMap<>();
-        for (String mdKey : metadata.keySet()) {
-            String[] values = metadata.getValues(mdKey);
-            // periods are not allowed - replace with %2E
-            mdKey = mdKey.replaceAll("\\.", "%2E");
-            metadataMap.put(mdKey, Arrays.asList(values));
-        }
-
-        String partitionKey = partitioner.getPartition(url, metadata);
-        if (partitionKey == null) {
-            partitionKey = "_DEFAULT_";
-        }
-
-        // send a tuple on the queue stream in case a bolt
-        // wants to handle it
-        super.collector.emit(
-                org.apache.stormcrawler.Constants.QUEUE_STREAM_NAME,
-                new Values(partitionKey, metadata));
-
-        // store routing key in metadata?
-        if (StringUtils.isNotBlank(fieldNameForRoutingKey) && routingFieldNameInMetadata) {
-            metadataMap.put(fieldNameForRoutingKey, partitionKey);
-        }
-
-        doc.put("metadata", metadataMap);
-
-        // store routing key outside metadata?
-        if (StringUtils.isNotBlank(fieldNameForRoutingKey) && !routingFieldNameInMetadata) {
-            doc.put(fieldNameForRoutingKey, partitionKey);
-        }
-
-        if (nextFetch.isPresent()) {
-            doc.put("nextFetchDate", nextFetch.get().toInstant().toString());
-        }
-        // check that we don't overwrite an existing entry
-        // When create is used, the index operation will fail if a document
-        // by that id already exists in the index.
-        final boolean create = status.equals(Status.DISCOVERED);
-        final String targetIndex = getIndexName(metadata);
-        final String routing = doRouting ? partitionKey : null;
-
-        BulkOperation op;
-        if (create) {
-            op =
-                    BulkOperation.of(
-                            b ->
-                                    b.create(
-                                            c -> {
-                                                c.index(targetIndex).id(documentID).document(doc);
-                                                if (routing != null) {
-                                                    c.routing(routing);
-                                                }
-                                                return c;
-                                            }));
-        } else {
-            op =
-                    BulkOperation.of(
-                            b ->
-                                    b.index(
-                                            idx -> {
-                                                idx.index(targetIndex).id(documentID).document(doc);
-                                                if (routing != null) {
-                                                    idx.routing(routing);
-                                                }
-                                                return idx;
-                                            }));
-        }
-
-        waitAck.addTuple(documentID, tuple);
-
-        LOG.debug("Sending to OpenSearch buffer {} with ID {}", url, documentID);
-
-        connection.addToProcessor(op);
-    }
-
-    @Override
-    public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
-        LOG.debug("afterBulk [{}] with {} responses", executionId, request.operations().size());
-        eventCounter.scope("bulks_received").incrBy(1);
-        eventCounter.scope("bulk_msec").incrBy(response.took());
-        eventCounter.scope("received").incrBy(request.operations().size());
-        receivedPerSecMetrics.scope("received").update(request.operations().size());
-
-        waitAck.processBulkResponse(
-                response,
-                executionId,
-                eventCounter,
-                (id, tuple, selected) -> {
-                    if (!selected.failed()) {
-                        String url = tuple.getStringByField("url");
-                        LOG.debug("Acked {} with ID {}", url, id);
-                        eventCounter.scope("acked").incrBy(1);
-                        super.ack(tuple, url);
-                    } else {
-                        eventCounter.scope("failed").incrBy(1);
-                        collector.fail(tuple);
-                    }
-                });
-    }
-
-    @Override
-    public void afterBulk(long executionId, BulkRequest request, Throwable throwable) {
-        eventCounter.scope("bulks_received").incrBy(1);
-        eventCounter.scope("received").incrBy(request.operations().size());
-        receivedPerSecMetrics.scope("received").update(request.operations().size());
-
-        waitAck.processFailedBulk(
-                request,
-                executionId,
-                throwable,
-                t -> {
-                    eventCounter.scope("failed").incrBy(1);
-                    collector.fail(t);
-                });
-    }
-
-    @Override
-    public void beforeBulk(long executionId, BulkRequest request) {
-        LOG.debug("beforeBulk {} with {} actions", executionId, request.operations().size());
-        eventCounter.scope("bulks_sent").incrBy(1);
-    }
-
-    /**
-     * Must be overridden for implementing custom index names based on some metadata information By
-     * Default, indexName coming from config is used
-     */
-    protected String getIndexName(Metadata m) {
-        return indexName;
-    }
-}
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/DelegateRefresherTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/DelegateRefresherTest.java
deleted file mode 100644
index 67bcca62..00000000
--- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/DelegateRefresherTest.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertInstanceOf;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.stormcrawler.JSONResource;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.filtering.URLFilter;
-import org.apache.stormcrawler.parse.ParseFilter;
-import org.apache.stormcrawler.parse.ParseResult;
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-import org.junit.jupiter.api.Test;
-import org.w3c.dom.DocumentFragment;
-
-class DelegateRefresherTest {
-
-    private static final ObjectMapper MAPPER = new ObjectMapper();
-
-    /** Minimal URLFilter + JSONResource implementation for testing. */
-    public static class StubURLFilter extends URLFilter implements JSONResource {
-
-        public final AtomicBoolean configured = new AtomicBoolean(false);
-
-        @Override
-        public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode params) {
-            configured.set(true);
-        }
-
-        @Override
-        public @Nullable String filter(
-                @Nullable URL sourceUrl,
-                @Nullable Metadata sourceMetadata,
-                @NotNull String urlToFilter) {
-            return urlToFilter;
-        }
-
-        @Override
-        public String getResourceFile() {
-            return "stub.json";
-        }
-
-        @Override
-        public void loadJSONResources(InputStream inputStream) throws IOException {}
-    }
-
-    /** Minimal ParseFilter + JSONResource implementation for testing. */
-    public static class StubParseFilter extends ParseFilter implements JSONResource {
-
-        public final AtomicBoolean configured = new AtomicBoolean(false);
-
-        @Override
-        public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode params) {
-            configured.set(true);
-        }
-
-        @Override
-        public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) {}
-
-        @Override
-        public String getResourceFile() {
-            return "stub.json";
-        }
-
-        @Override
-        public void loadJSONResources(InputStream inputStream) throws IOException {}
-    }
-
-    /** A URLFilter that does NOT implement JSONResource. */
-    public static class NonJsonResourceURLFilter extends URLFilter {
-
-        @Override
-        public @Nullable String filter(
-                @Nullable URL sourceUrl,
-                @Nullable Metadata sourceMetadata,
-                @NotNull String urlToFilter) {
-            return urlToFilter;
-        }
-    }
-
-    /** Not a URLFilter at all. */
-    public static class NotAFilter {}
-
-    private JsonNode buildParams(String delegateClass) {
-        return buildParams(delegateClass, 600);
-    }
-
-    private JsonNode buildParams(String delegateClass, int refreshRate) {
-        ObjectNode delegate = MAPPER.createObjectNode();
-        delegate.put("class", delegateClass);
-        delegate.set("params", MAPPER.createObjectNode());
-
-        ObjectNode params = MAPPER.createObjectNode();
-        params.set("delegate", delegate);
-        params.put("refresh", refreshRate);
-        return params;
-    }
-
-    @Test
-    void loadsURLFilterDelegate() {
-        JsonNode params = buildParams(StubURLFilter.class.getName());
-        Map<String, Object> conf = new HashMap<>();
-
-        DelegateRefresher<URLFilter> refresher =
-                new DelegateRefresher<>(
-                        URLFilter.class, conf, params, (d, c, p) -> d.configure(c, p));
-
-        try {
-            assertNotNull(refresher.getDelegate());
-            assertInstanceOf(StubURLFilter.class, refresher.getDelegate());
-            assertTrue(((StubURLFilter) refresher.getDelegate()).configured.get());
-        } finally {
-            refresher.cleanup();
-        }
-    }
-
-    @Test
-    void loadsParseFilterDelegate() {
-        JsonNode params = buildParams(StubParseFilter.class.getName());
-        Map<String, Object> conf = new HashMap<>();
-
-        DelegateRefresher<ParseFilter> refresher =
-                new DelegateRefresher<>(
-                        ParseFilter.class, conf, params, (d, c, p) -> d.configure(c, p));
-
-        try {
-            assertNotNull(refresher.getDelegate());
-            assertInstanceOf(StubParseFilter.class, refresher.getDelegate());
-            assertTrue(((StubParseFilter) refresher.getDelegate()).configured.get());
-        } finally {
-            refresher.cleanup();
-        }
-    }
-
-    @Test
-    void delegateFilterActuallyWorks() {
-        JsonNode params = buildParams(StubURLFilter.class.getName());
-        Map<String, Object> conf = new HashMap<>();
-
-        DelegateRefresher<URLFilter> refresher =
-                new DelegateRefresher<>(
-                        URLFilter.class, conf, params, (d, c, p) -> d.configure(c, p));
-
-        try {
-            String result = refresher.getDelegate().filter(null, null, "http://example.com");
-            assertEquals("http://example.com", result);
-        } finally {
-            refresher.cleanup();
-        }
-    }
-
-    @Test
-    void throwsWhenDelegateNodeMissing() {
-        ObjectNode params = MAPPER.createObjectNode();
-        // no "delegate" key
-        Map<String, Object> conf = new HashMap<>();
-
-        assertThrows(
-                RuntimeException.class,
-                () ->
-                        new DelegateRefresher<>(
-                                URLFilter.class, conf, params, (d, c, p) -> d.configure(c, p)));
-    }
-
-    @Test
-    void throwsWhenClassMissing() {
-        ObjectNode delegate = MAPPER.createObjectNode();
-        // no "class" key
-        ObjectNode params = MAPPER.createObjectNode();
-        params.set("delegate", delegate);
-        Map<String, Object> conf = new HashMap<>();
-
-        assertThrows(
-                RuntimeException.class,
-                () ->
-                        new DelegateRefresher<>(
-                                URLFilter.class, conf, params, (d, c, p) -> d.configure(c, p)));
-    }
-
-    @Test
-    void throwsWhenClassDoesNotExtendBaseType() {
-        JsonNode params = buildParams(NotAFilter.class.getName());
-        Map<String, Object> conf = new HashMap<>();
-
-        RuntimeException ex =
-                assertThrows(
-                        RuntimeException.class,
-                        () ->
-                                new DelegateRefresher<>(
-                                        URLFilter.class,
-                                        conf,
-                                        params,
-                                        (d, c, p) -> d.configure(c, p)));
-        assertTrue(ex.getMessage().contains("does not extend"));
-    }
-
-    @Test
-    void throwsWhenClassDoesNotImplementJSONResource() {
-        JsonNode params = buildParams(NonJsonResourceURLFilter.class.getName());
-        Map<String, Object> conf = new HashMap<>();
-
-        RuntimeException ex =
-                assertThrows(
-                        RuntimeException.class,
-                        () ->
-                                new DelegateRefresher<>(
-                                        URLFilter.class,
-                                        conf,
-                                        params,
-                                        (d, c, p) -> d.configure(c, p)));
-        assertTrue(ex.getMessage().contains("does not implement JSONResource"));
-    }
-
-    @Test
-    void cleanupIsIdempotent() {
-        JsonNode params = buildParams(StubURLFilter.class.getName());
-        Map<String, Object> conf = new HashMap<>();
-
-        DelegateRefresher<URLFilter> refresher =
-                new DelegateRefresher<>(
-                        URLFilter.class, conf, params, (d, c, p) -> d.configure(c, p));
-
-        // calling cleanup twice should not throw
-        refresher.cleanup();
-        refresher.cleanup();
-    }
-}
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java
deleted file mode 100644
index bb484e84..00000000
--- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.util.List;
-import org.apache.hc.client5.http.auth.AuthScope;
-import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
-import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
-import org.apache.hc.core5.http.HttpHost;
-import org.junit.jupiter.api.Test;
-
-class OpenSearchConnectionCredentialsTest {
-
-    private static BasicCredentialsProvider providerFor(List<HttpHost> hosts) {
-        final BasicCredentialsProvider provider = new BasicCredentialsProvider();
-        final UsernamePasswordCredentials credentials =
-                new UsernamePasswordCredentials("crawler", "s3cret".toCharArray());
-        for (AuthScope scope : OpenSearchConnection.credentialScopes(hosts)) {
-            provider.setCredentials(scope, credentials);
-        }
-        return provider;
-    }
-
-    @Test
-    void scopesCoverTheConfiguredHostsAndPorts() {
-        final List<AuthScope> scopes =
-                OpenSearchConnection.credentialScopes(
-                        List.of(
-                                new HttpHost("https", "opensearch1.example.org", 9200),
-                                new HttpHost("https", "opensearch2.example.org", 9201)));
-        assertEquals(
-                List.of(
-                        new AuthScope("https", "opensearch1.example.org", 9200, null, null),
-                        new AuthScope("https", "opensearch2.example.org", 9201, null, null)),
-                scopes);
-    }
-
-    @Test
-    void duplicateAddressesGiveOneScope() {
-        final List<AuthScope> scopes =
-                OpenSearchConnection.credentialScopes(
-                        List.of(
-                                new HttpHost("https", "opensearch1.example.org", 9200),
-                                new HttpHost("https", "OpenSearch1.example.org", 9200)));
-        assertEquals(1, scopes.size());
-    }
-
-    @Test
-    void noAddressGivesNoScope() {
-        assertTrue(OpenSearchConnection.credentialScopes(List.of()).isEmpty());
-    }
-
-    @Test
-    void credentialsAreOnlyGivenToTheConfiguredHosts() {
-        final BasicCredentialsProvider provider =
-                providerFor(List.of(new HttpHost("https", "opensearch1.example.org", 9200)));
-
-        assertNotNull(
-                provider.getCredentials(request("https", "opensearch1.example.org", 9200), null));
-        // another node of the cluster which is not listed in the addresses
-        assertNull(provider.getCredentials(request("https", "10.0.0.12", 9200), null));
-        // same host, another port
-        assertNull(
-                provider.getCredentials(request("https", "opensearch1.example.org", 9300), null));
-        assertNull(provider.getCredentials(request("https", "other.example.org", 9200), null));
-    }
-
-    @Test
-    void credentialsAreMatchedOnTheScheme() {
-        final BasicCredentialsProvider provider =
-                providerFor(List.of(new HttpHost("https", "opensearch1.example.org", 9200)));
-        assertNull(provider.getCredentials(request("http", "opensearch1.example.org", 9200), null));
-    }
-
-    /** The scope HttpClient looks the credentials up with for a request to the given node. */
-    private static AuthScope request(String scheme, String host, int port) {
-        return new AuthScope(new HttpHost(scheme, host, port), null, "Basic");
-    }
-
-    @Test
-    void plainHttpToARemoteHostIsReported() {
-        final HttpHost remote = new HttpHost("http", "opensearch1.example.org", 9200);
-        final List<HttpHost> plain =
-                OpenSearchConnection.plainHttpHosts(
-                        List.of(
-                                remote,
-                                new HttpHost("https", "opensearch2.example.org", 9200),
-                                new HttpHost("http", "localhost", 9200),
-                                new HttpHost("http", "127.0.0.1", 9200),
-                                new HttpHost("http", "[::1]", 9200)));
-        assertEquals(List.of(remote), plain);
-    }
-
-    @Test
-    void loopbackAddresses() {
-        assertTrue(OpenSearchConnection.isLoopback("localhost"));
-        assertTrue(OpenSearchConnection.isLoopback("LOCALHOST"));
-        assertTrue(OpenSearchConnection.isLoopback("127.0.0.1"));
-        assertTrue(OpenSearchConnection.isLoopback("[::1]"));
-        assertTrue(OpenSearchConnection.isLoopback("0:0:0:0:0:0:0:1"));
-        assertFalse(OpenSearchConnection.isLoopback("opensearch1.example.org"));
-        assertFalse(OpenSearchConnection.isLoopback("10.0.0.12"));
-        assertFalse(OpenSearchConnection.isLoopback("localhost.example.org"));
-        assertFalse(OpenSearchConnection.isLoopback("127.example.org"));
-        assertTrue(OpenSearchConnection.isLoopback("127.1.2.3"));
-    }
-}
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java
deleted file mode 100644
index 3bd2dc6d..00000000
--- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java
+++ /dev/null
@@ -1,336 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch;
-
-import static org.awaitility.Awaitility.await;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertSame;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.TimeUnit;
-import org.apache.storm.tuple.Tuple;
-import org.apache.stormcrawler.metrics.ScopedCounter;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.opensearch.client.opensearch._types.ErrorCause;
-import org.opensearch.client.opensearch.core.BulkRequest;
-import org.opensearch.client.opensearch.core.BulkResponse;
-import org.opensearch.client.opensearch.core.bulk.BulkOperation;
-import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
-import org.opensearch.client.opensearch.core.bulk.OperationType;
-import org.slf4j.LoggerFactory;
-
-class WaitAckCacheTest {
-
-    private WaitAckCache cache;
-    private List<Tuple> evicted;
-    private List<Tuple> acked;
-    private List<Tuple> failed;
-
-    @BeforeEach
-    void setUp() {
-        evicted = new CopyOnWriteArrayList<>();
-        acked = new ArrayList<>();
-        failed = new ArrayList<>();
-        cache = new WaitAckCache(LoggerFactory.getLogger(WaitAckCacheTest.class), evicted::add);
-    }
-
-    private Tuple mockTuple(String url) {
-        Tuple t = mock(Tuple.class);
-        when(t.getValueByField("url")).thenReturn(url);
-        when(t.getStringByField("url")).thenReturn(url);
-        return t;
-    }
-
-    private static BulkResponseItem successItem(String docId) {
-        return BulkResponseItem.of(
-                b -> b.id(docId).index("index").status(200).operationType(OperationType.Index));
-    }
-
-    private static BulkResponseItem failedItem(String docId, int status) {
-        return BulkResponseItem.of(
-                b ->
-                        b.id(docId)
-                                .index("index")
-                                .status(status)
-                                .operationType(OperationType.Index)
-                                .error(
-                                        ErrorCause.of(
-                                                e -> e.type("test_error").reason("test failure"))));
-    }
-
-    private static BulkResponse bulkResponse(BulkResponseItem... items) {
-        boolean hasErrors = false;
-        for (BulkResponseItem item : items) {
-            if (item.error() != null) {
-                hasErrors = true;
-                break;
-            }
-        }
-        final boolean errors = hasErrors;
-        return BulkResponse.of(b -> b.took(10).errors(errors).items(List.of(items)));
-    }
-
-    @Test
-    void addAndContains() {
-        Tuple t = mockTuple("http://example.com");
-        assertFalse(cache.contains("doc1"));
-
-        cache.addTuple("doc1", t);
-        assertTrue(cache.contains("doc1"));
-        assertEquals(1, cache.estimatedSize());
-    }
-
-    @Test
-    void invalidateRemovesEntry() {
-        Tuple t = mockTuple("http://example.com");
-        cache.addTuple("doc1", t);
-        assertTrue(cache.contains("doc1"));
-
-        cache.invalidate("doc1");
-        assertFalse(cache.contains("doc1"));
-    }
-
-    @Test
-    void processBulkResponse_successfulItem_ackedViaTupleAction() {
-        Tuple t = mockTuple("http://example.com");
-        cache.addTuple("doc1", t);
-
-        BulkResponse response = bulkResponse(successItem("doc1"));
-
-        cache.processBulkResponse(
-                response,
-                1L,
-                null,
-                (id, tuple, selected) -> {
-                    if (!selected.failed()) {
-                        acked.add(tuple);
-                    } else {
-                        failed.add(tuple);
-                    }
-                });
-
-        assertEquals(1, acked.size());
-        assertEquals(0, failed.size());
-        assertSame(t, acked.get(0));
-        assertFalse(cache.contains("doc1"));
-    }
-
-    @Test
-    void processBulkResponse_failedItem_failedViaTupleAction() {
-        Tuple t = mockTuple("http://example.com");
-        cache.addTuple("doc1", t);
-
-        BulkResponse response = bulkResponse(failedItem("doc1", 500));
-
-        cache.processBulkResponse(
-                response,
-                1L,
-                null,
-                (id, tuple, selected) -> {
-                    if (!selected.failed()) {
-                        acked.add(tuple);
-                    } else {
-                        failed.add(tuple);
-                    }
-                });
-
-        assertEquals(0, acked.size());
-        assertEquals(1, failed.size());
-        assertSame(t, failed.get(0));
-    }
-
-    @Test
-    void processBulkResponse_conflictIsNotAFailure() {
-        Tuple t = mockTuple("http://example.com");
-        cache.addTuple("doc1", t);
-
-        ScopedCounter counter = mock(ScopedCounter.class);
-        ScopedCounter.CountHandle handle = mock(ScopedCounter.CountHandle.class);
-        when(counter.scope("doc_conflicts")).thenReturn(handle);
-
-        BulkResponse response = bulkResponse(failedItem("doc1", 409));
-
-        cache.processBulkResponse(
-                response,
-                1L,
-                counter,
-                (id, tuple, selected) -> {
-                    if (!selected.failed()) {
-                        acked.add(tuple);
-                    } else {
-                        failed.add(tuple);
-                    }
-                });
-
-        assertEquals(1, acked.size());
-        assertEquals(0, failed.size());
-        verify(handle).incrBy(1);
-    }
-
-    @Test
-    void processBulkResponse_multipleTuplesForSameDocId() {
-        Tuple t1 = mockTuple("http://example.com/1");
-        Tuple t2 = mockTuple("http://example.com/2");
-        cache.addTuple("doc1", t1);
-        cache.addTuple("doc1", t2);
-
-        BulkResponse response = bulkResponse(successItem("doc1"));
-
-        cache.processBulkResponse(response, 1L, null, (id, tuple, selected) -> acked.add(tuple));
-
-        assertEquals(2, acked.size());
-        assertTrue(acked.contains(t1));
-        assertTrue(acked.contains(t2));
-    }
-
-    @Test
-    void processBulkResponse_duplicateDocIdInBulk_prefersSuccess() {
-        // https://github.com/apache/stormcrawler/issues/832
-        Tuple t = mockTuple("http://example.com");
-        cache.addTuple("doc1", t);
-
-        BulkResponse response = bulkResponse(failedItem("doc1", 500), successItem("doc1"));
-
-        cache.processBulkResponse(
-                response,
-                1L,
-                null,
-                (id, tuple, selected) -> {
-                    if (!selected.failed()) {
-                        acked.add(tuple);
-                    } else {
-                        failed.add(tuple);
-                    }
-                });
-
-        assertEquals(1, acked.size());
-        assertEquals(0, failed.size());
-    }
-
-    @Test
-    void processFailedBulk_failsAllMatchingTuples() {
-        Tuple t1 = mockTuple("http://example.com/1");
-        Tuple t2 = mockTuple("http://example.com/2");
-        cache.addTuple("doc1", t1);
-        cache.addTuple("doc2", t2);
-
-        BulkRequest request =
-                BulkRequest.of(
-                        b ->
-                                b.operations(
-                                        BulkOperation.of(
-                                                o -> o.delete(d -> d.index("index").id("doc1"))),
-                                        BulkOperation.of(
-                                                o -> o.delete(d -> d.index("index").id("doc2")))));
-
-        cache.processFailedBulk(request, 1L, new Exception("connection lost"), failed::add);
-
-        assertEquals(2, failed.size());
-        assertTrue(failed.contains(t1));
-        assertTrue(failed.contains(t2));
-        assertFalse(cache.contains("doc1"));
-        assertFalse(cache.contains("doc2"));
-    }
-
-    @Test
-    void processFailedBulk_ignoresMissingIds() {
-        Tuple t = mockTuple("http://example.com");
-        cache.addTuple("doc1", t);
-
-        BulkRequest request =
-                BulkRequest.of(
-                        b ->
-                                b.operations(
-                                        BulkOperation.of(
-                                                o ->
-                                                        o.delete(
-                                                                d ->
-                                                                        d.index("index")
-                                                                                .id(
-                                                                                        "doc_unknown")))));
-
-        cache.processFailedBulk(request, 1L, new Exception("test"), failed::add);
-
-        assertEquals(0, failed.size());
-        // doc1 should still be in cache since it wasn't in the failed request
-        assertTrue(cache.contains("doc1"));
-    }
-
-    @Test
-    void eviction_failsTuplesOnExpiry() {
-        cache =
-                new WaitAckCache(
-                        "expireAfterWrite=1s",
-                        LoggerFactory.getLogger(WaitAckCacheTest.class),
-                        evicted::add);
-        Tuple t = mockTuple("http://example.com");
-        cache.addTuple("doc1", t);
-
-        // Force cache maintenance after expiry by doing a contains() check
-        // which accesses the cache and triggers Caffeine's cleanup
-        await().atMost(5, TimeUnit.SECONDS)
-                .pollInterval(200, TimeUnit.MILLISECONDS)
-                .untilAsserted(
-                        () -> {
-                            // contains() accesses the cache which triggers cleanup
-                            cache.contains("doc1");
-                            // also try adding and invalidating a dummy entry to force maintenance
-                            Tuple dummy = mockTuple("http://dummy");
-                            cache.addTuple("_probe_", dummy);
-                            cache.invalidate("_probe_");
-                            assertFalse(evicted.isEmpty(), "Eviction callback should have fired");
-                        });
-
-        assertTrue(evicted.contains(t));
-    }
-
-    @Test
-    void processBulkResponse_multipleDocIds() {
-        Tuple t1 = mockTuple("http://example.com/1");
-        Tuple t2 = mockTuple("http://example.com/2");
-        cache.addTuple("doc1", t1);
-        cache.addTuple("doc2", t2);
-
-        BulkResponse response = bulkResponse(successItem("doc1"), failedItem("doc2", 500));
-
-        cache.processBulkResponse(
-                response,
-                1L,
-                null,
-                (id, tuple, selected) -> {
-                    if (!selected.failed()) {
-                        acked.add(tuple);
-                    } else {
-                        failed.add(tuple);
-                    }
-                });
-
-        assertEquals(1, acked.size());
-        assertSame(t1, acked.get(0));
-        assertEquals(1, failed.size());
-        assertSame(t2, failed.get(0));
-    }
-}
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
deleted file mode 100644
index 44150ea6..00000000
--- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.bolt;
-
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.junit.jupiter.Testcontainers;
-import org.testcontainers.utility.DockerImageName;
-
-@Testcontainers(disabledWithoutDocker = true)
-public abstract class AbstractOpenSearchTest {
-
-    private static final String OPENSEARCH_VERSION =
-            System.getProperty("opensearch-version", "2.19.5");
-
-    public static final String PASSWORD = "This1sAPassw0rd";
-
-    protected GenericContainer<?> opensearchContainer =
-            new GenericContainer<>(
-                            DockerImageName.parse(
-                                    "opensearchproject/opensearch:" + OPENSEARCH_VERSION))
-                    .withExposedPorts(9200)
-                    .withEnv("plugins.security.disabled", "true")
-                    .withEnv("discovery.type", "single-node")
-                    .withEnv("OPENSEARCH_JAVA_OPTS", "-Xms512m -Xmx512m")
-                    .withEnv("OPENSEARCH_INITIAL_ADMIN_PASSWORD", PASSWORD);
-
-    @BeforeEach
-    void init() {
-        opensearchContainer.start();
-    }
-
-    @AfterEach
-    void close() {
-        opensearchContainer.close();
-    }
-}
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/IndexerBoltTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/IndexerBoltTest.java
deleted file mode 100644
index aa953a28..00000000
--- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/IndexerBoltTest.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.bolt;
-
-import static org.awaitility.Awaitility.await;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.tuple.Tuple;
-import org.apache.stormcrawler.Constants;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.TestOutputCollector;
-import org.apache.stormcrawler.TestUtil;
-import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-class IndexerBoltTest extends AbstractOpenSearchTest {
-
-    private IndexerBolt bolt;
-
-    protected TestOutputCollector output;
-
-    private static final Logger LOG = LoggerFactory.getLogger(IndexerBoltTest.class);
-
-    private static ExecutorService executorService;
-
-    @BeforeAll
-    static void beforeClass() {
-        executorService = Executors.newFixedThreadPool(2);
-    }
-
-    @AfterAll
-    static void afterClass() {
-        executorService.shutdown();
-        executorService = null;
-    }
-
-    @BeforeEach
-    void setupIndexerBolt() {
-        bolt = new IndexerBolt("content");
-        // give the indexer the port for connecting to OpenSearch
-        final String host = opensearchContainer.getHost();
-        final Integer port = opensearchContainer.getFirstMappedPort();
-        final Map<String, Object> conf = new HashMap<>();
-        conf.put(AbstractIndexerBolt.urlFieldParamName, "url");
-        conf.put(AbstractIndexerBolt.canonicalMetadataParamName, "canonical");
-        conf.put("opensearch.indexer.addresses", host + ":" + port);
-        output = new TestOutputCollector();
-        bolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
-    }
-
-    @AfterEach
-    void close() {
-        LOG.info("Closing indexer bolt and Opensearch container");
-        super.close();
-        bolt.cleanup();
-        output = null;
-    }
-
-    private void index(String url, String text, Metadata metadata) {
-        Tuple tuple = mock(Tuple.class);
-        when(tuple.getStringByField("text")).thenReturn(text);
-        when(tuple.getStringByField("url")).thenReturn(url);
-        when(tuple.getValueByField("metadata")).thenReturn(metadata);
-        bolt.execute(tuple);
-    }
-
-    private int lastIndex(String url, String text, Metadata metadata, long timeoutInMs)
-            throws ExecutionException, InterruptedException, TimeoutException {
-        var oldSize = output.getEmitted(Constants.StatusStreamName).size();
-        index(url, text, metadata);
-        return executorService
-                .submit(
-                        () -> {
-                            await().atMost(timeoutInMs, TimeUnit.MILLISECONDS)
-                                    .until(
-                                            () ->
-                                                    output.getEmitted(Constants.StatusStreamName)
-                                                                    .size()
-                                                            > oldSize);
-                            return output.getEmitted(Constants.StatusStreamName).size();
-                        })
-                .get(timeoutInMs, TimeUnit.MILLISECONDS);
-    }
-
-    @Test
-    @Timeout(value = 2, unit = TimeUnit.MINUTES)
-    // https://github.com/apache/stormcrawler/issues/832
-    void simultaneousCanonicals()
-            throws ExecutionException, InterruptedException, TimeoutException {
-        Metadata m1 = new Metadata();
-        String url =
-                "https://www.obozrevatel.com/ukr/dnipro/city/u-dnipri-ta-oblasti-ogolosili-shtormove-poperedzhennya.htm";
-        m1.addValue("canonical", url);
-        Metadata m2 = new Metadata();
-        String url2 =
-                "https://www.obozrevatel.com/ukr/dnipro/city/u-dnipri-ta-oblasti-ogolosili-shtormove-poperedzhennya/amp.htm";
-        m2.addValue("canonical", url);
-        index(url, "", m1);
-        lastIndex(url2, "", m2, 10_000);
-        // should be two in status output
-        assertEquals(2, output.getEmitted(Constants.StatusStreamName).size());
-        // and 2 acked
-        assertEquals(2, output.getAckedTuples().size());
-        // TODO check output in Opensearch?
-    }
-}
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java
deleted file mode 100644
index 296c58dd..00000000
--- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.bolt;
-
-import static org.awaitility.Awaitility.await;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import org.apache.hc.core5.http.HttpHost;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.tuple.Tuple;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.TestOutputCollector;
-import org.apache.stormcrawler.TestUtil;
-import org.apache.stormcrawler.opensearch.persistence.QueueBolt;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.Timeout;
-import org.opensearch.client.json.jackson.JacksonJsonpMapper;
-import org.opensearch.client.opensearch.OpenSearchClient;
-import org.opensearch.client.opensearch.core.GetResponse;
-import org.opensearch.client.transport.OpenSearchTransport;
-import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-class QueueBoltTest extends AbstractOpenSearchTest {
-
-    private QueueBolt bolt;
-
-    protected TestOutputCollector output;
-
-    protected OpenSearchClient client;
-
-    private OpenSearchTransport transport;
-
-    private static final Logger LOG = LoggerFactory.getLogger(QueueBoltTest.class);
-
-    private static ExecutorService executorService;
-
-    @BeforeAll
-    static void beforeClass() {
-        executorService = Executors.newFixedThreadPool(2);
-    }
-
-    @AfterAll
-    static void afterClass() {
-        executorService.shutdown();
-        executorService = null;
-    }
-
-    @BeforeEach
-    void setupQueueBolt() throws IOException {
-        transport =
-                ApacheHttpClient5TransportBuilder.builder(
-                                new HttpHost(
-                                        "http",
-                                        opensearchContainer.getHost(),
-                                        opensearchContainer.getMappedPort(9200)))
-                        .setMapper(new JacksonJsonpMapper())
-                        .build();
-        client = new OpenSearchClient(transport);
-        output = new TestOutputCollector();
-        bolt = prepareQueueBolt();
-    }
-
-    private QueueBolt prepareQueueBolt() {
-        QueueBolt queueBolt = new QueueBolt();
-        Map<String, Object> conf = new HashMap<>();
-        conf.put(
-                "opensearch.queues.addresses",
-                opensearchContainer.getHost() + ":" + opensearchContainer.getFirstMappedPort());
-        queueBolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
-        return queueBolt;
-    }
-
-    @AfterEach
-    void close() {
-        LOG.info("Closing queue bolt and Opensearch container");
-        super.close();
-        bolt.cleanup();
-        output = null;
-        try {
-            transport.close();
-        } catch (IOException e) {
-        }
-    }
-
-    private Future<Integer> store(String key, Metadata metadata) {
-        Tuple tuple = mock(Tuple.class);
-        when(tuple.getStringByField("key")).thenReturn(key);
-        when(tuple.getValueByField("metadata")).thenReturn(metadata);
-        bolt.execute(tuple);
-        return executorService.submit(
-                () -> {
-                    await().atMost(30, TimeUnit.SECONDS)
-                            .until(() -> output.getAckedTuples().size() > 0);
-                    return output.getAckedTuples().size();
-                });
-    }
-
-    private void awaitIndexed(String key) {
-        String id = org.apache.commons.codec.digest.DigestUtils.sha256Hex(key);
-        await().atMost(30, TimeUnit.SECONDS)
-                .until(
-                        () -> {
-                            try {
-                                GetResponse<Map> result =
-                                        client.get(g -> g.index("queues").id(id), Map.class);
-                                return result.found();
-                            } catch (IOException e) {
-                                return false;
-                            }
-                        });
-    }
-
-    @SuppressWarnings("unchecked")
-    private Map<String, Object> getSource(String key) throws IOException {
-        String id = org.apache.commons.codec.digest.DigestUtils.sha256Hex(key);
-        return client.get(g -> g.index("queues").id(id), Map.class).source();
-    }
-
-    @Test
-    @Timeout(value = 2, unit = TimeUnit.MINUTES)
-    void checkQueueIndex()
-            throws IOException, ExecutionException, InterruptedException, TimeoutException {
-        String key = "www.url.net";
-        Metadata md = new Metadata();
-        md.addValue("someKey", "someValue");
-        store(key, md).get(10, TimeUnit.SECONDS);
-        assertEquals(1, output.getAckedTuples().size());
-
-        // Wait until document is indexed in OpenSearch
-        awaitIndexed(key);
-
-        Map<String, Object> sourceAsMap = getSource(key);
-        assertEquals(key, sourceAsMap.get("key"));
-    }
-
-    @Test
-    @Timeout(value = 2, unit = TimeUnit.MINUTES)
-    void checkExistingEntryIsNotOverwritten()
-            throws IOException, ExecutionException, InterruptedException, TimeoutException {
-        String key = "www.url.net";
-        Metadata md = new Metadata();
-        md.addValue("someKey", "someValue");
-        store(key, md).get(10, TimeUnit.SECONDS);
-        awaitIndexed(key);
-        Object lastUpdated = getSource(key).get("lastUpdated");
-
-        // a fresh bolt instance (e.g. after a worker restart) has an empty cache
-        // and re-sends the same key - the resulting version conflict must be
-        // handled gracefully and the existing entry left untouched
-        QueueBolt restartedBolt = prepareQueueBolt();
-        try {
-            Tuple sameKey = mock(Tuple.class);
-            when(sameKey.getStringByField("key")).thenReturn(key);
-            restartedBolt.execute(sameKey);
-
-            // a new key sent in the same bulk signals when the bulk has been processed
-            String otherKey = "www.other.net";
-            Tuple other = mock(Tuple.class);
-            when(other.getStringByField("key")).thenReturn(otherKey);
-            restartedBolt.execute(other);
-            awaitIndexed(otherKey);
-
-            Map<String, Object> sourceAsMap = getSource(key);
-            assertEquals(key, sourceAsMap.get("key"));
-            assertEquals(lastUpdated, sourceAsMap.get("lastUpdated"));
-        } finally {
-            restartedBolt.cleanup();
-        }
-    }
-}
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java
deleted file mode 100644
index 71fe7795..00000000
--- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.bolt;
-
-import static org.awaitility.Awaitility.await;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import org.apache.hc.core5.http.HttpHost;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.tuple.Tuple;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.TestOutputCollector;
-import org.apache.stormcrawler.TestUtil;
-import org.apache.stormcrawler.opensearch.persistence.StatusUpdaterBolt;
-import org.apache.stormcrawler.persistence.Status;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.Timeout;
-import org.opensearch.client.json.jackson.JacksonJsonpMapper;
-import org.opensearch.client.opensearch.OpenSearchClient;
-import org.opensearch.client.opensearch.core.GetResponse;
-import org.opensearch.client.transport.OpenSearchTransport;
-import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-class StatusBoltTest extends AbstractOpenSearchTest {
-
-    private StatusUpdaterBolt bolt;
-
-    protected TestOutputCollector output;
-
-    protected OpenSearchClient client;
-
-    private OpenSearchTransport transport;
-
-    private static final Logger LOG = LoggerFactory.getLogger(StatusBoltTest.class);
-
-    private static ExecutorService executorService;
-
-    @BeforeAll
-    static void beforeClass() {
-        executorService = Executors.newFixedThreadPool(2);
-    }
-
-    @AfterAll
-    static void afterClass() {
-        executorService.shutdown();
-        executorService = null;
-    }
-
-    @BeforeEach
-    void setupStatusBolt() throws IOException {
-        bolt = new StatusUpdaterBolt();
-        transport =
-                ApacheHttpClient5TransportBuilder.builder(
-                                new HttpHost(
-                                        "http",
-                                        opensearchContainer.getHost(),
-                                        opensearchContainer.getMappedPort(9200)))
-                        .setMapper(new JacksonJsonpMapper())
-                        .build();
-        client = new OpenSearchClient(transport);
-        // configure the status updater bolt
-        Map<String, Object> conf = new HashMap<>();
-        conf.put("opensearch.status.routing.fieldname", "metadata.key");
-        conf.put(
-                "opensearch.status.addresses",
-                opensearchContainer.getHost() + ":" + opensearchContainer.getFirstMappedPort());
-        conf.put("scheduler.class", "org.apache.stormcrawler.persistence.DefaultScheduler");
-        conf.put("status.updater.cache.spec", "maximumSize=10000,expireAfterAccess=1h");
-        conf.put("metadata.persist", "someKey");
-        output = new TestOutputCollector();
-        bolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
-    }
-
-    @AfterEach
-    void close() {
-        LOG.info("Closing updater bolt and Opensearch container");
-        super.close();
-        bolt.cleanup();
-        output = null;
-        try {
-            transport.close();
-        } catch (IOException e) {
-        }
-    }
-
-    private Future<Integer> store(String url, Status status, Metadata metadata) {
-        Tuple tuple = mock(Tuple.class);
-        when(tuple.getValueByField("status")).thenReturn(status);
-        when(tuple.getStringByField("url")).thenReturn(url);
-        when(tuple.getValueByField("metadata")).thenReturn(metadata);
-        bolt.execute(tuple);
-        return executorService.submit(
-                () -> {
-                    await().atMost(30, TimeUnit.SECONDS)
-                            .until(() -> output.getAckedTuples().size() > 0);
-                    return output.getAckedTuples().size();
-                });
-    }
-
-    @Test
-    @Timeout(value = 2, unit = TimeUnit.MINUTES)
-    // see https://github.com/apache/stormcrawler/issues/885
-    @SuppressWarnings("unchecked")
-    void checkListKeyFromOpensearch()
-            throws IOException, ExecutionException, InterruptedException, TimeoutException {
-        String url = "https://www.url.net/something";
-        Metadata md = new Metadata();
-        md.addValue("someKey", "someValue");
-        store(url, Status.DISCOVERED, md).get(10, TimeUnit.SECONDS);
-        assertEquals(1, output.getAckedTuples().size());
-        // check output in Opensearch
-        String id = org.apache.commons.codec.digest.DigestUtils.sha256Hex(url);
-        GetResponse<Map> result = client.get(g -> g.index("status").id(id), Map.class);
-        Map<String, Object> sourceAsMap = result.source();
-        final String pfield = "metadata.somekey";
-        sourceAsMap = (Map<String, Object>) sourceAsMap.get("metadata");
-        final var pfieldNew = pfield.substring(9);
-        Object key = sourceAsMap.get(pfieldNew);
-        assertTrue(key instanceof java.util.ArrayList);
-    }
-
-    @Test
-    @Timeout(value = 2, unit = TimeUnit.MINUTES)
-    void checkQueueStreamEmission()
-            throws IOException, ExecutionException, InterruptedException, TimeoutException {
-        String url = "https://www.url.net/something";
-        Metadata md = new Metadata();
-        md.addValue("someKey", "someValue");
-        store(url, Status.DISCOVERED, md).get(10, TimeUnit.SECONDS);
-        assertEquals(1, output.getAckedTuples().size());
-
-        // Check that a tuple was emitted on the queue stream
-        var emittedQueue = output.getEmitted(org.apache.stormcrawler.Constants.QUEUE_STREAM_NAME);
-        assertEquals(1, emittedQueue.size());
-
-        var queueTuple = emittedQueue.get(0);
-        assertEquals(2, queueTuple.size());
-        assertEquals("www.url.net", queueTuple.get(0));
-        assertTrue(queueTuple.get(1) instanceof Metadata);
-        Metadata emittedMetadata = (Metadata) queueTuple.get(1);
-        assertEquals("someValue", emittedMetadata.getFirstValue("someKey"));
-    }
-}
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporterTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporterTest.java
deleted file mode 100644
index 8c0c9413..00000000
--- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporterTest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.metrics;
-
-import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.MetricRegistry;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.stormcrawler.opensearch.bolt.AbstractOpenSearchTest;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.Timeout;
-
-class MetricsReporterTest extends AbstractOpenSearchTest {
-
-    @Test
-    @Timeout(60)
-    void prepareAndReportMetrics() {
-        MetricRegistry registry = new MetricRegistry();
-        Counter counter = registry.counter("test.counter");
-        counter.inc(42);
-
-        Map<String, Object> topoConf = new HashMap<>();
-        topoConf.put(
-                "opensearch.metrics.addresses",
-                opensearchContainer.getHost() + ":" + opensearchContainer.getFirstMappedPort());
-
-        Map<String, Object> reporterConf = new HashMap<>();
-        reporterConf.put("report.period", 60L);
-        reporterConf.put("report.period.units", "SECONDS");
-
-        MetricsReporter reporter = new MetricsReporter();
-        assertDoesNotThrow(() -> reporter.prepare(registry, topoConf, reporterConf));
-        assertNotNull(reporter);
-        reporter.stop();
-    }
-}
diff --git a/external/opensearch-java/src/test/resources/indexer.mapping b/external/opensearch-java/src/test/resources/indexer.mapping
deleted file mode 100644
index fc6eb887..00000000
--- a/external/opensearch-java/src/test/resources/indexer.mapping
+++ /dev/null
@@ -1,40 +0,0 @@
-{
-	"settings": {
-		"index": {
-			"number_of_shards": 5,
-			"number_of_replicas": 1,
-			"refresh_interval": "60s"
-		}
-	},
-	"mappings": {
-			"_source": {
-				"enabled": true
-			},
-			"properties": {
-				"content": {
-					"type": "text"
-				},
-				"description": {
-					"type": "text"
-				},
-				"domain": {
-					"type": "keyword"
-				},
-				"format": {
-					"type": "keyword"
-				},
-				"keywords": {
-					"type": "keyword"
-				},
-				"host": {
-					"type": "keyword"
-				},
-				"title": {
-					"type": "text"
-				},
-				"url": {
-					"type": "keyword"
-				}
-			}
-	}
-}
diff --git a/external/opensearch-java/src/test/resources/metrics.mapping b/external/opensearch-java/src/test/resources/metrics.mapping
deleted file mode 100644
index fc6ae3a0..00000000
--- a/external/opensearch-java/src/test/resources/metrics.mapping
+++ /dev/null
@@ -1,40 +0,0 @@
-{
-	  "index_patterns": "metrics*",
-	  "settings": {
-	    "index": {
-	      "number_of_shards": 1,
-	      "refresh_interval": "30s"
-	    },
-	    "number_of_replicas": 0
-	  },
-	  "mappings": {
-	      "_source":         { "enabled": true },
-	      "properties": {
-	          "name": {
-	            "type": "keyword"
-	          },
-	          "stormId": {
-	            "type": "keyword"
-	          },
-	          "srcComponentId": {
-	            "type": "keyword"
-	          },
-	          "srcTaskId": {
-	            "type": "short"
-	          },
-	          "srcWorkerHost": {
-	            "type": "keyword"
-	          },
-	          "srcWorkerPort": {
-	            "type": "integer"
-	          },
-	          "timestamp": {
-	            "type": "date",
-	            "format": "date_optional_time"
-	          },
-	          "value": {
-	            "type": "double"
-	          }
-	      }
-	  }
-}
diff --git a/external/opensearch-java/src/test/resources/queues.mapping b/external/opensearch-java/src/test/resources/queues.mapping
deleted file mode 100644
index b85f33dc..00000000
--- a/external/opensearch-java/src/test/resources/queues.mapping
+++ /dev/null
@@ -1,33 +0,0 @@
-{
-	"settings": {
-		"index": {
-			"number_of_shards": 10,
-			"number_of_replicas": 1,
-			"refresh_interval": "5s"
-		}
-	},
-    "mappings": {
-            "dynamic_templates": [{
-                "metadata": {
-                    "path_match": "metadata.*",
-                    "match_mapping_type": "string",
-                    "mapping": {
-                        "type": "keyword"
-                    }
-                }
-            }],
-			"_source": {
-				"enabled": true
-			},
-			"properties": {
-				"key": {
-					"type": "keyword",
-					"index": true
-				},
-				"lastUpdated": {
-					"type": "date",
-					"format": "date_optional_time"
-				}
-			}
-	}
-}
diff --git a/external/opensearch-java/src/test/resources/status.mapping b/external/opensearch-java/src/test/resources/status.mapping
deleted file mode 100644
index e5b14fe9..00000000
--- a/external/opensearch-java/src/test/resources/status.mapping
+++ /dev/null
@@ -1,39 +0,0 @@
-{
-	"settings": {
-		"index": {
-			"number_of_shards": 10,
-			"number_of_replicas": 1,
-			"refresh_interval": "5s"
-		}
-	},
-	"mappings": {
-			"dynamic_templates": [{
-				"metadata": {
-					"path_match": "metadata.*",
-					"match_mapping_type": "string",
-					"mapping": {
-						"type": "keyword"
-					}
-				}
-			}],
-			"_source": {
-				"enabled": true
-			},
-			"properties": {
-				"key": {
-					"type": "keyword",
-					"index": true
-				},
-				"nextFetchDate": {
-					"type": "date",
-					"format": "date_optional_time"
-				},
-				"status": {
-					"type": "keyword"
-				},
-				"url": {
-					"type": "keyword"
-				}
-			}
-	}
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java
similarity index 100%
rename from external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java
rename to external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
index e4eec09e..c32f162f 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
@@ -17,118 +17,55 @@
 
 package org.apache.stormcrawler.opensearch;
 
-import java.io.IOException;
+import java.util.Objects;
 import org.jetbrains.annotations.NotNull;
-import org.opensearch.action.DocWriteRequest;
-import org.opensearch.action.DocWriteResponse;
-import org.opensearch.action.bulk.BulkItemResponse;
-import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.core.rest.RestStatus;
-import org.opensearch.core.xcontent.ToXContent;
-import org.opensearch.core.xcontent.XContentBuilder;
+import org.jetbrains.annotations.Nullable;
+import org.opensearch.client.opensearch._types.ErrorCause;
+import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
 
-public final class BulkItemResponseToFailedFlag {
-    @NotNull public final BulkItemResponse response;
-    public final boolean failed;
-    @NotNull public final String id;
+/**
+ * Wraps a {@link BulkResponseItem} with a pre-computed failure flag. A 409 (conflict) is not
+ * considered a failure — it simply indicates a document already existed when using create mode.
+ *
+ * @param response the original bulk response item
+ * @param failed whether this item represents a real failure (excludes 409 conflicts)
+ * @param id the document id from the response item
+ */
+public record BulkItemResponseToFailedFlag(
+        @NotNull BulkResponseItem response, boolean failed, @NotNull String id) {
 
-    public BulkItemResponseToFailedFlag(@NotNull BulkItemResponse response, boolean failed) {
-        this.response = response;
-        this.failed = failed;
-        this.id = response.getId();
+    public BulkItemResponseToFailedFlag {
+        Objects.requireNonNull(response, "response");
+        Objects.requireNonNull(id, "id");
     }
 
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (!(o instanceof BulkItemResponseToFailedFlag)) {
-            return false;
-        }
-
-        BulkItemResponseToFailedFlag that = (BulkItemResponseToFailedFlag) o;
-
-        if (failed != that.failed) {
-            return false;
-        }
-        if (!response.equals(that.response)) {
-            return false;
-        }
-        return id.equals(that.id);
+    /** Constructs with id derived from the response item. */
+    public BulkItemResponseToFailedFlag(@NotNull BulkResponseItem response, boolean failed) {
+        this(
+                response,
+                failed,
+                Objects.requireNonNull(response.id(), "BulkResponseItem id must not be null"));
     }
 
-    @Override
-    public int hashCode() {
-        int result = response.hashCode();
-        result = 31 * result + (failed ? 1 : 0);
-        result = 31 * result + id.hashCode();
-        return result;
+    /** Returns the error cause, or {@code null} if the item did not fail. */
+    @Nullable
+    public ErrorCause getFailedCause() {
+        return response.error();
     }
 
-    @Override
-    public String toString() {
-        return "BulkItemResponseToFailedFlag{"
-                + "response="
-                + response
-                + ", failed="
-                + failed
-                + ", id='"
-                + id
-                + '\''
-                + '}';
+    /** Returns a human-readable failure description, or {@code null} if the item did not fail. */
+    @Nullable
+    public String getFailure() {
+        ErrorCause error = response.error();
+        if (error == null) {
+            return null;
+        }
+        return error.reason() != null ? error.reason() : error.type();
     }
 
-    public RestStatus status() {
+    // opensearch-java: status() returns int HTTP code, not RestStatus enum
+    /** Returns the HTTP status code of this response item. */
+    public int getStatus() {
         return response.status();
     }
-
-    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params)
-            throws IOException {
-        return response.toXContent(builder, params);
-    }
-
-    public int getItemId() {
-        return response.getItemId();
-    }
-
-    public DocWriteRequest.OpType getOpType() {
-        return response.getOpType();
-    }
-
-    public String getIndex() {
-        return response.getIndex();
-    }
-
-    public long getVersion() {
-        return response.getVersion();
-    }
-
-    public <T extends DocWriteResponse> T getResponse() {
-        return response.getResponse();
-    }
-
-    public boolean isFailed() {
-        return response.isFailed();
-    }
-
-    public String getFailureMessage() {
-        return response.getFailureMessage();
-    }
-
-    public BulkItemResponse.Failure getFailure() {
-        return response.getFailure();
-    }
-
-    public void writeTo(StreamOutput out) throws IOException {
-        response.writeTo(out);
-    }
-
-    public void writeThin(StreamOutput out) throws IOException {
-        response.writeThin(out);
-    }
-
-    public boolean isFragment() {
-        return response.isFragment();
-    }
 }
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java
index 4935bc1a..fb03a9f0 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java
@@ -20,14 +20,14 @@ package org.apache.stormcrawler.opensearch;
 import com.fasterxml.jackson.databind.JsonNode;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.Map;
 import java.util.Timer;
 import java.util.TimerTask;
 import org.apache.stormcrawler.JSONResource;
-import org.opensearch.action.get.GetRequest;
-import org.opensearch.action.get.GetResponse;
-import org.opensearch.client.RequestOptions;
-import org.opensearch.client.RestHighLevelClient;
+import org.opensearch.client.json.JsonData;
+import org.opensearch.client.opensearch.OpenSearchClient;
+import org.opensearch.client.opensearch.core.GetResponse;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -38,6 +38,10 @@ import org.slf4j.LoggerFactory;
  * org.apache.stormcrawler.opensearch.parse.filter.JSONResourceWrapper} to eliminate duplicated
  * setup/refresh/cleanup logic.
  *
+ * <p>This is the opensearch-java (OpenSearch Java Client 3.x / HC5) counterpart of the class with
+ * the same name in the {@code external/opensearch} module. It uses the typed {@link
+ * OpenSearchClient} instead of the deprecated {@code RestHighLevelClient}.
+ *
  * @param <T> the base type that the delegate must extend (e.g. URLFilter or ParseFilter)
  */
 public class DelegateRefresher<T> {
@@ -46,7 +50,7 @@ public class DelegateRefresher<T> {
 
     private final T delegate;
     private Timer refreshTimer;
-    private RestHighLevelClient osClient;
+    private OpenSearchClient osClient;
 
     /**
      * Creates a refresher by loading the delegate class from the JSON configuration.
@@ -127,13 +131,18 @@ public class DelegateRefresher<T> {
                         if (osClient != null) {
                             LOG.info("Reloading json resources from OpenSearch");
                             try {
-                                GetResponse response =
+                                GetResponse<JsonData> response =
                                         osClient.get(
-                                                new GetRequest(
-                                                        "config", resource.getResourceFile()),
-                                                RequestOptions.DEFAULT);
-                                resource.loadJSONResources(
-                                        new ByteArrayInputStream(response.getSourceAsBytes()));
+                                                g ->
+                                                        g.index("config")
+                                                                .id(resource.getResourceFile()),
+                                                JsonData.class);
+                                if (response.found() && response.source() != null) {
+                                    String json = response.source().toJson().toString();
+                                    resource.loadJSONResources(
+                                            new ByteArrayInputStream(
+                                                    json.getBytes(StandardCharsets.UTF_8)));
+                                }
                             } catch (Exception e) {
                                 LOG.error("Can't load config from OpenSearch", e);
                             }
@@ -156,7 +165,7 @@ public class DelegateRefresher<T> {
         }
         if (osClient != null) {
             try {
-                osClient.close();
+                osClient._transport().close();
             } catch (IOException e) {
                 LOG.error("Exception when closing OpenSearch client", e);
             }
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java
index 180a1074..f7617205 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java
@@ -17,29 +17,22 @@
 
 package org.apache.stormcrawler.opensearch;
 
-import com.google.common.base.Charsets;
 import com.google.common.io.Resources;
 import java.io.IOException;
 import java.net.URL;
-import org.opensearch.OpenSearchException;
-import org.opensearch.action.support.master.AcknowledgedResponse;
-import org.opensearch.client.RequestOptions;
-import org.opensearch.client.RestHighLevelClient;
-import org.opensearch.client.indices.CreateIndexRequest;
-import org.opensearch.client.indices.CreateIndexResponse;
-import org.opensearch.client.indices.GetIndexRequest;
-import org.opensearch.client.indices.IndexTemplatesExistRequest;
-import org.opensearch.client.indices.PutIndexTemplateRequest;
-import org.opensearch.common.xcontent.XContentType;
+import java.nio.charset.StandardCharsets;
+import org.opensearch.client.opensearch.OpenSearchClient;
+import org.opensearch.client.opensearch.generic.Requests;
+import org.opensearch.client.opensearch.generic.Response;
+import org.opensearch.client.opensearch.indices.ExistsTemplateRequest;
 import org.slf4j.Logger;
 
 public class IndexCreation {
 
     public static synchronized void checkOrCreateIndex(
-            RestHighLevelClient client, String indexName, String boltType, Logger log)
+            OpenSearchClient client, String indexName, String boltType, Logger log)
             throws IOException {
-        final boolean indexExists =
-                client.indices().exists(new GetIndexRequest(indexName), RequestOptions.DEFAULT);
+        final boolean indexExists = client.indices().exists(req -> req.index(indexName)).value();
         log.info("Index '{}' exists? {}", indexName, indexExists);
         // there's a possible check-then-update race condition
         // createIndex intentionally catches and logs exceptions from OpenSearch
@@ -51,13 +44,12 @@ public class IndexCreation {
     }
 
     public static synchronized void checkOrCreateIndexTemplate(
-            RestHighLevelClient client, String boltType, Logger log) throws IOException {
+            OpenSearchClient client, String boltType, Logger log) throws IOException {
         final String templateName = boltType + "-template";
         final boolean templateExists =
                 client.indices()
-                        .existsTemplate(
-                                new IndexTemplatesExistRequest(templateName),
-                                RequestOptions.DEFAULT);
+                        .existsTemplate(ExistsTemplateRequest.of(r -> r.name(templateName)))
+                        .value();
         log.info("Template '{}' exists? {}", templateName, templateExists);
         // there's a possible check-then-update race condition
         // createTemplate intentionally catches and logs exceptions from OpenSearch
@@ -69,46 +61,54 @@ public class IndexCreation {
     }
 
     private static boolean createTemplate(
-            RestHighLevelClient client, String templateName, String resourceName, Logger log) {
+            OpenSearchClient client, String templateName, String resourceName, Logger log) {
 
         try {
-            final PutIndexTemplateRequest createIndexRequest =
-                    new PutIndexTemplateRequest(templateName);
-
             final URL mapping =
                     Thread.currentThread().getContextClassLoader().getResource(resourceName);
 
-            final String jsonIndexConfiguration = Resources.toString(mapping, Charsets.UTF_8);
-
-            createIndexRequest.source(jsonIndexConfiguration, XContentType.JSON);
-
-            final AcknowledgedResponse createIndexResponse =
-                    client.indices().putTemplate(createIndexRequest, RequestOptions.DEFAULT);
-            return createIndexResponse.isAcknowledged();
-        } catch (IOException | OpenSearchException e) {
+            final String jsonIndexConfiguration =
+                    Resources.toString(mapping, StandardCharsets.UTF_8);
+
+            try (Response response =
+                    client.generic()
+                            .execute(
+                                    Requests.builder()
+                                            .endpoint("/_template/" + templateName)
+                                            .method("PUT")
+                                            .json(jsonIndexConfiguration)
+                                            .build())) {
+                int statusCode = response.getStatus();
+                return statusCode == 200 || statusCode == 201;
+            }
+        } catch (Exception e) {
             log.warn("template '{}' not created", templateName, e);
             return false;
         }
     }
 
     private static boolean createIndex(
-            RestHighLevelClient client, String indexName, String resourceName, Logger log) {
+            OpenSearchClient client, String indexName, String resourceName, Logger log) {
 
         try {
-
-            final CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
-
             final URL mapping =
                     Thread.currentThread().getContextClassLoader().getResource(resourceName);
 
-            final String jsonIndexConfiguration = Resources.toString(mapping, Charsets.UTF_8);
-
-            createIndexRequest.source(jsonIndexConfiguration, XContentType.JSON);
-
-            final CreateIndexResponse createIndexResponse =
-                    client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
-            return createIndexResponse.isAcknowledged();
-        } catch (IOException | OpenSearchException e) {
+            final String jsonIndexConfiguration =
+                    Resources.toString(mapping, StandardCharsets.UTF_8);
+
+            try (Response response =
+                    client.generic()
+                            .execute(
+                                    Requests.builder()
+                                            .endpoint("/" + indexName)
+                                            .method("PUT")
+                                            .json(jsonIndexConfiguration)
+                                            .build())) {
+                int statusCode = response.getStatus();
+                return statusCode == 200 || statusCode == 201;
+            }
+        } catch (Exception e) {
             log.warn("index '{}' not created", indexName, e);
             return false;
         }
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
index f9a5dea1..eeb47362 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
@@ -17,8 +17,8 @@
 
 package org.apache.stormcrawler.opensearch;
 
-import static org.opensearch.client.RestClientBuilder.DEFAULT_CONNECT_TIMEOUT_MILLIS;
-import static org.opensearch.client.RestClientBuilder.DEFAULT_SOCKET_TIMEOUT_MILLIS;
+import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_CONNECT_TIMEOUT_MILLIS;
+import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_RESPONSE_TIMEOUT_MILLIS;
 
 import java.io.IOException;
 import java.net.URI;
@@ -30,32 +30,29 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.regex.Pattern;
+import javax.net.ssl.SSLContext;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.HttpHost;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.Credentials;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
-import org.apache.http.conn.ssl.TrustAllStrategy;
-import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.hc.client5.http.auth.AuthScope;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.config.ConnectionConfig;
+import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
+import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
+import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
+import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.reactor.ssl.TlsDetails;
+import org.apache.hc.core5.ssl.SSLContextBuilder;
+import org.apache.hc.core5.util.Timeout;
 import org.apache.stormcrawler.util.ConfUtils;
 import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-import org.opensearch.action.DocWriteRequest;
-import org.opensearch.action.bulk.BulkProcessor;
-import org.opensearch.action.bulk.BulkRequest;
-import org.opensearch.action.bulk.BulkResponse;
-import org.opensearch.client.HttpAsyncResponseConsumerFactory;
-import org.opensearch.client.Node;
-import org.opensearch.client.RequestOptions;
-import org.opensearch.client.RestClient;
-import org.opensearch.client.RestClientBuilder;
-import org.opensearch.client.RestHighLevelClient;
-import org.opensearch.client.sniff.OpenSearchNodesSniffer;
-import org.opensearch.client.sniff.Sniffer;
-import org.opensearch.common.unit.TimeValue;
+import org.opensearch.client.json.jackson.JacksonJsonpMapper;
+import org.opensearch.client.opensearch.OpenSearchAsyncClient;
+import org.opensearch.client.opensearch.OpenSearchClient;
+import org.opensearch.client.opensearch.core.bulk.BulkOperation;
+import org.opensearch.client.transport.OpenSearchTransport;
+import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -66,24 +63,248 @@ public final class OpenSearchConnection {
 
     private static final Logger LOG = LoggerFactory.getLogger(OpenSearchConnection.class);
 
-    @NotNull private final RestHighLevelClient client;
+    @NotNull private final OpenSearchClient client;
 
-    @NotNull private final BulkProcessor processor;
+    @NotNull private final AsyncBulkProcessor processor;
 
-    @Nullable private final Sniffer sniffer;
+    @NotNull private final OpenSearchTransport transport;
 
     private OpenSearchConnection(
-            @NotNull RestHighLevelClient c, @NotNull BulkProcessor p, @Nullable Sniffer s) {
-        processor = p;
+            @NotNull OpenSearchClient c,
+            @NotNull AsyncBulkProcessor p,
+            @NotNull OpenSearchTransport t) {
         client = c;
-        sniffer = s;
+        processor = p;
+        transport = t;
     }
 
-    public RestHighLevelClient getClient() {
+    public OpenSearchClient getClient() {
         return client;
     }
 
-    public static RestHighLevelClient getClient(Map<String, Object> stormConf, String boltType) {
+    /**
+     * Creates a standalone {@link OpenSearchClient}. Used by classes that need a client without a
+     * bulk processor (e.g. spouts, filters). Callers are responsible for closing the returned
+     * client's transport via {@code client._transport().close()}.
+     */
+    public static OpenSearchClient getClient(Map<String, Object> stormConf, String boltType) {
+        return buildClientResources(stormConf, boltType).client();
+    }
+
+    /** Adds a single bulk operation to the internal processor. */
+    public void addToProcessor(final BulkOperation operation) {
+        processor.add(operation);
+    }
+
+    /**
+     * Creates a connection with a default (no-op) listener. The values for bolt type are
+     * [indexer,status,metrics].
+     */
+    public static OpenSearchConnection getConnection(
+            Map<String, Object> stormConf, String boltType) {
+        AsyncBulkProcessor.Listener listener =
+                new AsyncBulkProcessor.Listener() {
+                    @Override
+                    public void afterBulk(
+                            long arg0,
+                            org.opensearch.client.opensearch.core.BulkRequest arg1,
+                            org.opensearch.client.opensearch.core.BulkResponse arg2) {}
+
+                    @Override
+                    public void afterBulk(
+                            long arg0,
+                            org.opensearch.client.opensearch.core.BulkRequest arg1,
+                            Throwable arg2) {}
+
+                    @Override
+                    public void beforeBulk(
+                            long arg0, org.opensearch.client.opensearch.core.BulkRequest arg1) {}
+                };
+        return getConnection(stormConf, boltType, listener);
+    }
+
+    public static OpenSearchConnection getConnection(
+            Map<String, Object> stormConf, String boltType, AsyncBulkProcessor.Listener listener) {
+
+        final String dottedType = boltType + ".";
+
+        warnOnRemovedKeys(stormConf, dottedType);
+
+        ClientResources cr = buildClientResources(stormConf, boltType);
+
+        final String flushIntervalString =
+                ConfUtils.getString(
+                        stormConf, Constants.PARAMPREFIX, dottedType, "flushInterval", "5s");
+
+        final long flushIntervalMillis = parseTimeValueToMillis(flushIntervalString, 5000);
+
+        final int bulkActions =
+                ConfUtils.getInt(stormConf, Constants.PARAMPREFIX, dottedType, "bulkActions", 50);
+
+        final int concurrentRequests =
+                ConfUtils.getInt(
+                        stormConf, Constants.PARAMPREFIX, dottedType, "concurrentRequests", 1);
+
+        AsyncBulkProcessor bulkProcessor = null;
+        try {
+            bulkProcessor =
+                    new AsyncBulkProcessor.Builder(
+                                    new OpenSearchAsyncClient(cr.transport()), listener)
+                            .setBulkActions(bulkActions)
+                            .setFlushIntervalMillis(flushIntervalMillis)
+                            .setConcurrentRequests(concurrentRequests)
+                            .build();
+
+            return new OpenSearchConnection(cr.client(), bulkProcessor, cr.transport());
+        } catch (Exception e) {
+            if (bulkProcessor != null) {
+                try {
+                    bulkProcessor.close();
+                } catch (Exception suppressed) {
+                    e.addSuppressed(suppressed);
+                }
+            }
+            try {
+                cr.transport().close();
+            } catch (IOException suppressed) {
+                e.addSuppressed(suppressed);
+            }
+            throw e;
+        }
+    }
+
+    private final AtomicBoolean isClosed = new AtomicBoolean(false);
+
+    public void close() {
+
+        if (!isClosed.compareAndSet(false, true)) {
+            LOG.warn("Tried to close an already closed connection!");
+            return;
+        }
+
+        LOG.debug("Start closing the OpenSearch connection");
+
+        // First, close the BulkProcessor ensuring pending actions are flushed
+        try {
+            boolean success = processor.awaitClose(60, TimeUnit.SECONDS);
+            if (!success) {
+                throw new RuntimeException(
+                        "Failed to flush pending actions when closing BulkProcessor");
+            }
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+
+        // Now close the transport (also shuts down the underlying HTTP client)
+        try {
+            transport.close();
+        } catch (IOException e) {
+            LOG.trace("Transport threw IO exception on close.");
+        }
+    }
+
+    /**
+     * Extracts the document ID from a {@link BulkOperation} regardless of its type (index, create,
+     * delete, update).
+     */
+    public static String getBulkOperationId(BulkOperation op) {
+        if (op.isIndex()) {
+            return op.index().id();
+        }
+        if (op.isCreate()) {
+            return op.create().id();
+        }
+        if (op.isDelete()) {
+            return op.delete().id();
+        }
+        if (op.isUpdate()) {
+            return op.update().id();
+        }
+        return null;
+    }
+
+    /**
+     * Returns the scopes the Basic credentials are registered for: the scheme, host and port of
+     * each configured address. A request to any other address does not receive them.
+     */
+    static List<AuthScope> credentialScopes(List<HttpHost> hosts) {
+        final Set<AuthScope> scopes = new LinkedHashSet<>();
+        for (HttpHost host : hosts) {
+            scopes.add(new AuthScope(host));
+        }
+        return new ArrayList<>(scopes);
+    }
+
+    /**
+     * Returns the addresses which use plain http and are not a loopback address, i.e. those the
+     * Basic credentials would be sent to in the clear over the network.
+     */
+    static List<HttpHost> plainHttpHosts(List<HttpHost> hosts) {
+        final List<HttpHost> plain = new ArrayList<>();
+        for (HttpHost host : hosts) {
+            if ("http".equalsIgnoreCase(host.getSchemeName()) && !isLoopback(host.getHostName())) {
+                plain.add(host);
+            }
+        }
+        return plain;
+    }
+
+    private static final Pattern IPV4_LOOPBACK = Pattern.compile("127(\\.\\d{1,3}){3}");
+
+    /** Whether the host is localhost or a loopback IP literal. No name is resolved. */
+    static boolean isLoopback(String hostname) {
+        String host = StringUtils.strip(hostname.toLowerCase(Locale.ROOT), "[]");
+        return host.equals("localhost")
+                || IPV4_LOOPBACK.matcher(host).matches()
+                || host.equals("::1")
+                || host.equals("0:0:0:0:0:0:0:1");
+    }
+
+    private static void warnAboutPlainHttp(String boltType, List<HttpHost> hosts) {
+        final List<HttpHost> plain = plainHttpHosts(hosts);
+        if (!plain.isEmpty()) {
+            LOG.warn(
+                    "OpenSearch credentials are configured for {} but the addresses {} use plain "
+                            + "http, the credentials are sent unencrypted. Give the addresses an "
+                            + "https:// scheme.",
+                    boltType,
+                    plain);
+        }
+    }
+
+    // internal helpers
+    private record ClientResources(OpenSearchClient client, OpenSearchTransport transport) {}
+
+    /**
+     * Logs a WARN for legacy configuration keys that are no longer honoured by this module, so that
+     * users migrating from {@code external/opensearch} notice silently-dropped tuning. See the
+     * module README for the full list of differences.
+     */
+    private static void warnOnRemovedKeys(Map<String, Object> stormConf, String dottedType) {
+        final String responseBufferKey = Constants.PARAMPREFIX + dottedType + "responseBufferSize";
+        if (stormConf.containsKey(responseBufferKey)) {
+            LOG.warn(
+                    "Configuration key '{}' is set but no longer supported by the opensearch-java module. "
+                            + "The HC5-based async transport does not expose an equivalent per-request "
+                            + "heap-buffer override. The setting is ignored — remove it from your "
+                            + "configuration. See external/opensearch-java/README.md for details.",
+                    responseBufferKey);
+        }
+
+        final String sniffKey = Constants.PARAMPREFIX + dottedType + "sniff";
+        if (stormConf.containsKey(sniffKey)) {
+            LOG.warn(
+                    "Configuration key '{}' is set but no longer supported by the opensearch-java module. "
+                            + "The OpenSearch Java Client 3.x does not ship a Sniffer equivalent, so "
+                            + "automatic node discovery is not available. Keep the 'addresses' list up to "
+                            + "date manually or put a load balancer in front of the cluster. "
+                            + "See external/opensearch-java/README.md for details.",
+                    sniffKey);
+        }
+    }
+
+    private static ClientResources buildClientResources(
+            Map<String, Object> stormConf, String boltType) {
 
         final String dottedType = boltType + ".";
 
@@ -119,10 +340,15 @@ public final class OpenSearchConnection {
             if (uri.getScheme() != null) {
                 scheme = uri.getScheme();
             }
-            hosts.add(new HttpHost(uri.getHost(), port, scheme));
+            // HC5: constructor is (scheme, hostname, port) — not (hostname, port, scheme)
+            hosts.add(new HttpHost(scheme, uri.getHost(), port));
         }
 
-        final RestClientBuilder builder = RestClient.builder(hosts.toArray(new HttpHost[0]));
+        LOG.info(
+                "OpenSearch {} transport configured with {} host(s): {}",
+                boltType,
+                hosts.size(),
+                hosts);
 
         // authentication via user / password
         final String user =
@@ -144,9 +370,40 @@ public final class OpenSearchConnection {
                 ConfUtils.getBoolean(
                         stormConf, Constants.PARAMPREFIX, "", "disable.tls.validation", false);
 
-        final boolean needsUser = hasCredentials(user, password);
+        final boolean needsUser = StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password);
         final boolean needsProxy = StringUtils.isNotBlank(proxyhost) && proxyport != -1;
 
+        // Defaults from ApacheHttpClient5TransportBuilder (same as the former RestClientBuilder)
+        final int connectTimeout =
+                ConfUtils.getInt(
+                        stormConf,
+                        Constants.PARAMPREFIX,
+                        dottedType,
+                        "connect.timeout",
+                        DEFAULT_CONNECT_TIMEOUT_MILLIS);
+        final int socketTimeout =
+                ConfUtils.getInt(
+                        stormConf,
+                        Constants.PARAMPREFIX,
+                        dottedType,
+                        "socket.timeout",
+                        DEFAULT_RESPONSE_TIMEOUT_MILLIS);
+
+        final boolean compression =
+                ConfUtils.getBoolean(
+                        stormConf, Constants.PARAMPREFIX, dottedType, "compression", false);
+
+        final ApacheHttpClient5TransportBuilder builder =
+                ApacheHttpClient5TransportBuilder.builder(hosts.toArray(new HttpHost[0]))
+                        .setMapper(new JacksonJsonpMapper());
+
+        // Timeouts via ConnectionConfig on the builder's internal connection manager
+        builder.setConnectionConfigCallback(
+                connConfigBuilder ->
+                        connConfigBuilder
+                                .setConnectTimeout(Timeout.ofMilliseconds(connectTimeout))
+                                .setSocketTimeout(Timeout.ofMilliseconds(socketTimeout)));
+
         if (needsUser) {
             warnAboutPlainHttp(boltType, hosts);
         }
@@ -157,27 +414,61 @@ public final class OpenSearchConnection {
                     boltType);
         }
 
+        // Auth, proxy, and/or trust-all SSL via HttpClient customisation
         if (needsUser || needsProxy || disableTlsValidation) {
             builder.setHttpClientConfigCallback(
                     httpClientBuilder -> {
+                        // hc.client5 auth: password is char[], AuthScope(host, port)
                         if (needsUser) {
-                            httpClientBuilder.setDefaultCredentialsProvider(
-                                    new OriginCredentialsProvider(
-                                            hosts,
-                                            new UsernamePasswordCredentials(user, password)));
+                            final BasicCredentialsProvider credentialsProvider =
+                                    new BasicCredentialsProvider();
+                            final UsernamePasswordCredentials credentials =
+                                    new UsernamePasswordCredentials(user, password.toCharArray());
+                            for (AuthScope scope : credentialScopes(hosts)) {
+                                credentialsProvider.setCredentials(scope, credentials);
+                            }
+                            httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
                         }
+                        // hc.client5 proxy: HttpHost(scheme, host, port)
                         if (needsProxy) {
                             httpClientBuilder.setProxy(
-                                    new HttpHost(proxyhost, proxyport, proxyscheme));
+                                    new HttpHost(proxyscheme, proxyhost, proxyport));
                         }
-
+                        // Custom connection manager overrides the builder's internal one,
+                        // so timeouts and TlsDetailsFactory must be replicated here
                         if (disableTlsValidation) {
                             try {
-                                final SSLContextBuilder sslContext = new SSLContextBuilder();
-                                sslContext.loadTrustMaterial(null, new TrustAllStrategy());
-                                httpClientBuilder.setSSLContext(sslContext.build());
-                                httpClientBuilder.setSSLHostnameVerifier(
-                                        NoopHostnameVerifier.INSTANCE);
+                                final SSLContext sslContext =
+                                        SSLContextBuilder.create()
+                                                .loadTrustMaterial((chain, authType) -> true)
+                                                .build();
+                                httpClientBuilder.setConnectionManager(
+                                        PoolingAsyncClientConnectionManagerBuilder.create()
+                                                .setTlsStrategy(
+                                                        ClientTlsStrategyBuilder.create()
+                                                                .setSslContext(sslContext)
+                                                                .setHostnameVerifier(
+                                                                        NoopHostnameVerifier
+                                                                                .INSTANCE)
+                                                                // HTTP/2 ALPN negotiation
+                                                                .setTlsDetailsFactory(
+                                                                        sslEngine ->
+                                                                                new TlsDetails(
+                                                                                        sslEngine
+                                                                                                .getSession(),
+                                                                                        sslEngine
+                                                                                                .getApplicationProtocol()))
+                                                                .build())
+                                                .setDefaultConnectionConfig(
+                                                        ConnectionConfig.custom()
+                                                                .setConnectTimeout(
+                                                                        Timeout.ofMilliseconds(
+                                                                                connectTimeout))
+                                                                .setSocketTimeout(
+                                                                        Timeout.ofMilliseconds(
+                                                                                socketTimeout))
+                                                                .build())
+                                                .build());
                             } catch (Exception e) {
                                 throw new RuntimeException("Failed to disable TLS validation", e);
                             }
@@ -186,324 +477,40 @@ public final class OpenSearchConnection {
                     });
         }
 
-        final int connectTimeout =
-                ConfUtils.getInt(
-                        stormConf,
-                        Constants.PARAMPREFIX,
-                        dottedType,
-                        "connect.timeout",
-                        DEFAULT_CONNECT_TIMEOUT_MILLIS);
-        final int socketTimeout =
-                ConfUtils.getInt(
-                        stormConf,
-                        Constants.PARAMPREFIX,
-                        dottedType,
-                        "socket.timeout",
-                        DEFAULT_SOCKET_TIMEOUT_MILLIS);
-        // timeout until connection is established
-        builder.setRequestConfigCallback(
-                requestConfigBuilder ->
-                        requestConfigBuilder
-                                .setConnectTimeout(connectTimeout)
-                                // Timeout when waiting for data
-                                .setSocketTimeout(socketTimeout));
-
-        // TODO check if this has gone somewhere else
-        // int maxRetryTimeout = ConfUtils.getInt(stormConf, Constants.PARAMPREFIX +
-        // boltType +
-        // ".max.retry.timeout",
-        // DEFAULT_MAX_RETRY_TIMEOUT_MILLIS);
-        // builder.setMaxRetryTimeoutMillis(maxRetryTimeout);
-
-        // TODO configure headers etc...
-        // Map<String, String> configSettings = (Map) stormConf
-        // .get(Constants.PARAMPREFIX + boltType + ".settings");
-        // if (configSettings != null) {
-        // configSettings.forEach((k, v) -> settings.put(k, v));
-        // }
-
-        // use node selector only to log nodes listed in the config
-        // and/or discovered through sniffing
-        builder.setNodeSelector(
-                nodes -> {
-                    for (Node node : nodes) {
-                        LOG.debug(
-                                "Connected to OpenSearch node {} [{}] for {}",
-                                node.getName(),
-                                node.getHost(),
-                                boltType);
-                    }
-                });
-
-        final boolean compression =
-                ConfUtils.getBoolean(
-                        stormConf, Constants.PARAMPREFIX, dottedType, "compression", false);
-
+        // Compression: first-class builder method, not a request interceptor
         builder.setCompressionEnabled(compression);
 
-        return new RestHighLevelClient(builder);
-    }
+        final OpenSearchTransport transport = builder.build();
+        final OpenSearchClient openSearchClient = new OpenSearchClient(transport);
 
-    /** Basic authentication is only set up when both the user and the password are given. */
-    static boolean hasCredentials(String user, String password) {
-        return StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password);
+        return new ClientResources(openSearchClient, transport);
     }
 
     /**
-     * Gives the Basic credentials only to requests for one of the configured addresses, matched on
-     * scheme, host and port. A node reached under any other address, such as one found by the
-     * sniffer under the address it publishes, does not receive them.
+     * Parses a time value string (e.g. "5s", "500ms", "1m") into milliseconds.
+     *
+     * @param value the string to parse
+     * @param defaultMillis the default if parsing fails
+     * @return milliseconds
      */
-    static final class OriginCredentialsProvider implements CredentialsProvider {
-
-        private final Set<String> origins = new LinkedHashSet<>();
-        private final Credentials credentials;
-
-        OriginCredentialsProvider(List<HttpHost> hosts, Credentials credentials) {
-            for (HttpHost host : hosts) {
-                origins.add(origin(host));
-            }
-            this.credentials = credentials;
-        }
-
-        private static String origin(HttpHost host) {
-            final String scheme = host.getSchemeName().toLowerCase(Locale.ROOT);
-            int port = host.getPort();
-            if (port < 0) {
-                port = "https".equals(scheme) ? 443 : 80;
-            }
-            return scheme + "://" + host.getHostName().toLowerCase(Locale.ROOT) + ":" + port;
-        }
-
-        @Override
-        public Credentials getCredentials(AuthScope scope) {
-            final HttpHost origin = scope.getOrigin();
-            if (origin != null && origins.contains(origin(origin))) {
-                return credentials;
-            }
-            return null;
+    static long parseTimeValueToMillis(String value, long defaultMillis) {
+        if (value == null || value.isBlank()) {
+            return defaultMillis;
         }
-
-        @Override
-        public void setCredentials(AuthScope scope, Credentials credentials) {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public void clear() {}
-    }
-
-    /**
-     * Returns the addresses which use plain http and are not a loopback address, i.e. those the
-     * Basic credentials would be sent to in the clear over the network.
-     */
-    static List<HttpHost> plainHttpHosts(List<HttpHost> hosts) {
-        final List<HttpHost> plain = new ArrayList<>();
-        for (HttpHost host : hosts) {
-            if ("http".equalsIgnoreCase(host.getSchemeName()) && !isLoopback(host.getHostName())) {
-                plain.add(host);
-            }
-        }
-        return plain;
-    }
-
-    /**
-     * Returns the scheme under which the sniffer registers the nodes it finds. The nodes publish no
-     * scheme, so https is used as soon as one configured address uses it; otherwise sniffing would
-     * move the requests to plain http after the first round.
-     */
-    static OpenSearchNodesSniffer.Scheme sniffScheme(List<HttpHost> hosts) {
-        for (HttpHost host : hosts) {
-            if ("https".equalsIgnoreCase(host.getSchemeName())) {
-                return OpenSearchNodesSniffer.Scheme.HTTPS;
-            }
-        }
-        return OpenSearchNodesSniffer.Scheme.HTTP;
-    }
-
-    private static final Pattern IPV4_LOOPBACK = Pattern.compile("127(\\.\\d{1,3}){3}");
-
-    /** Whether the host is localhost or a loopback IP literal. No name is resolved. */
-    static boolean isLoopback(String hostname) {
-        String host = StringUtils.strip(hostname.toLowerCase(Locale.ROOT), "[]");
-        return host.equals("localhost")
-                || IPV4_LOOPBACK.matcher(host).matches()
-                || host.equals("::1")
-                || host.equals("0:0:0:0:0:0:0:1");
-    }
-
-    private static void warnAboutPlainHttp(String boltType, List<HttpHost> hosts) {
-        final List<HttpHost> plain = plainHttpHosts(hosts);
-        if (!plain.isEmpty()) {
-            LOG.warn(
-                    "OpenSearch credentials are configured for {} but the addresses {} use plain "
-                            + "http, the credentials are sent unencrypted. Give the addresses an "
-                            + "https:// scheme.",
-                    boltType,
-                    plain);
-        }
-    }
-
-    public void addToProcessor(final DocWriteRequest<?> request) {
-        processor.add(request);
-    }
-
-    /**
-     * Creates a connection with a default listener. The values for bolt type are
-     * [indexer,status,metrics]
-     */
-    public static OpenSearchConnection getConnection(
-            Map<String, Object> stormConf, String boltType) {
-        BulkProcessor.Listener listener =
-                new BulkProcessor.Listener() {
-                    @Override
-                    public void afterBulk(long arg0, BulkRequest arg1, BulkResponse arg2) {}
-
-                    @Override
-                    public void afterBulk(long arg0, BulkRequest arg1, Throwable arg2) {}
-
-                    @Override
-                    public void beforeBulk(long arg0, BulkRequest arg1) {}
-                };
-        return getConnection(stormConf, boltType, listener);
-    }
-
-    public static OpenSearchConnection getConnection(
-            Map<String, Object> stormConf, String boltType, BulkProcessor.Listener listener) {
-
-        final RestHighLevelClient client = getClient(stormConf, boltType);
-
-        final String dottedType = boltType + ".";
-
-        final String flushIntervalString =
-                ConfUtils.getString(
-                        stormConf, Constants.PARAMPREFIX, dottedType, "flushInterval", "5s");
-
-        final TimeValue flushInterval =
-                TimeValue.parseTimeValue(
-                        flushIntervalString, TimeValue.timeValueSeconds(5), "flushInterval");
-
-        final int bulkActions =
-                ConfUtils.getInt(stormConf, Constants.PARAMPREFIX, dottedType, "bulkActions", 50);
-
-        final int concurrentRequests =
-                ConfUtils.getInt(
-                        stormConf, Constants.PARAMPREFIX, dottedType, "concurrentRequests", 1);
-
-        final RequestOptions requestOptions = RequestOptions.DEFAULT;
-        final RequestOptions.Builder requestOptionsBuilder = requestOptions.toBuilder();
-        final int bufferSize =
-                ConfUtils.getInt(
-                        stormConf, Constants.PARAMPREFIX, dottedType, "responseBufferSize", 100);
-
-        requestOptionsBuilder.setHttpAsyncResponseConsumerFactory(
-                new HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory(
-                        bufferSize * 1024 * 1024));
-
-        BulkProcessor bulkProcessor = null;
-        Sniffer sniffer = null;
-        try {
-            bulkProcessor =
-                    BulkProcessor.builder(
-                                    (request, bulkListener) ->
-                                            client.bulkAsync(
-                                                    request,
-                                                    requestOptionsBuilder.build(),
-                                                    bulkListener),
-                                    listener)
-                            .setFlushInterval(flushInterval)
-                            .setBulkActions(bulkActions)
-                            .setConcurrentRequests(concurrentRequests)
-                            .build();
-
-            boolean sniff =
-                    ConfUtils.getBoolean(
-                            stormConf, Constants.PARAMPREFIX, dottedType, "sniff", true);
-            if (sniff) {
-                if (hasCredentials(
-                        ConfUtils.getString(stormConf, Constants.PARAMPREFIX, dottedType, "user"),
-                        ConfUtils.getString(
-                                stormConf, Constants.PARAMPREFIX, dottedType, "password"))) {
-                    LOG.warn(
-                            "Sniffing is enabled for {} and OpenSearch credentials are configured. "
-                                    + "The credentials are only sent to the configured addresses: "
-                                    + "requests to a node the sniffer finds under another host or "
-                                    + "port are sent without them and fail if the cluster requires "
-                                    + "authentication. List every node in opensearch.{}.addresses "
-                                    + "or set opensearch.{}.sniff: false.",
-                            boltType,
-                            boltType,
-                            boltType);
-                }
-                final RestClient lowLevelClient = client.getLowLevelClient();
-                final List<HttpHost> configured = new ArrayList<>();
-                for (Node node : lowLevelClient.getNodes()) {
-                    configured.add(node.getHost());
-                }
-                sniffer =
-                        Sniffer.builder(lowLevelClient)
-                                .setNodesSniffer(
-                                        new OpenSearchNodesSniffer(
-                                                lowLevelClient,
-                                                OpenSearchNodesSniffer
-                                                        .DEFAULT_SNIFF_REQUEST_TIMEOUT,
-                                                sniffScheme(configured)))
-                                .build();
-            }
-
-            return new OpenSearchConnection(client, bulkProcessor, sniffer);
-        } catch (Exception e) {
-            if (bulkProcessor != null) {
-                try {
-                    bulkProcessor.close();
-                } catch (Exception suppressed) {
-                    e.addSuppressed(suppressed);
-                }
-            }
-            try {
-                client.close();
-            } catch (IOException suppressed) {
-                e.addSuppressed(suppressed);
-            }
-            throw e;
-        }
-    }
-
-    private boolean isClosed = false;
-
-    public void close() {
-
-        if (isClosed) {
-            LOG.warn("Tried to close an already closed connection!");
-            return;
-        }
-
-        // Maybe some kind of identifier?
-        LOG.debug("Start closing the OpenSearch connection");
-
-        // First, close the BulkProcessor ensuring pending actions are flushed
+        value = value.trim();
         try {
-            boolean success = processor.awaitClose(60, TimeUnit.SECONDS);
-            if (!success) {
-                throw new RuntimeException(
-                        "Failed to flush pending actions when closing BulkProcessor");
+            if (value.endsWith("ms")) {
+                return Long.parseLong(value.substring(0, value.length() - 2));
+            } else if (value.endsWith("s")) {
+                return Long.parseLong(value.substring(0, value.length() - 1)) * 1000;
+            } else if (value.endsWith("m")) {
+                return Long.parseLong(value.substring(0, value.length() - 1)) * 60000;
+            } else {
+                return Long.parseLong(value);
             }
-        } catch (InterruptedException e) {
-            throw new RuntimeException(e);
-        }
-
-        if (sniffer != null) {
-            sniffer.close();
+        } catch (NumberFormatException e) {
+            LOG.warn("Could not parse time value '{}', using default {}ms", value, defaultMillis);
+            return defaultMillis;
         }
-
-        // Now close the actual client
-        try {
-            client.close();
-        } catch (IOException e) {
-            // ignore silently
-            LOG.trace("Client threw IO exception.");
-        }
-
-        isClosed = true;
     }
 }
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java
index bdb9a8dc..0e8574fb 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java
@@ -21,23 +21,22 @@ import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.Caffeine;
 import com.github.benmanes.caffeine.cache.RemovalCause;
 import com.github.benmanes.caffeine.cache.Ticker;
-import java.util.Arrays;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 import org.apache.storm.tuple.Tuple;
 import org.apache.stormcrawler.metrics.ScopedCounter;
 import org.jetbrains.annotations.Nullable;
-import org.opensearch.action.DocWriteRequest;
-import org.opensearch.action.bulk.BulkItemResponse;
-import org.opensearch.action.bulk.BulkRequest;
-import org.opensearch.action.bulk.BulkResponse;
-import org.opensearch.core.rest.RestStatus;
+// opensearch-java: uses typed BulkRequest/BulkResponse, not legacy REST equivalents
+import org.opensearch.client.opensearch.core.BulkRequest;
+import org.opensearch.client.opensearch.core.BulkResponse;
 import org.slf4j.Logger;
 
 /**
@@ -54,8 +53,7 @@ public class WaitAckCache {
     }
 
     private final Cache<String, List<Tuple>> cache;
-    private final java.util.concurrent.locks.ReentrantLock lock =
-            new java.util.concurrent.locks.ReentrantLock(true);
+    private final ReentrantLock lock = new ReentrantLock(true);
     private final Logger log;
     private final Consumer<Tuple> onEviction;
 
@@ -64,7 +62,10 @@ public class WaitAckCache {
         this(Caffeine.newBuilder().expireAfterWrite(60, TimeUnit.SECONDS), log, onEviction);
     }
 
-    /** Creates a cache from a Caffeine spec string (e.g. "expireAfterWrite=300s"). */
+    /**
+     * Creates a cache using a {@link Caffeine} spec string (e.g. {@code "expireAfterWrite=300s"}),
+     * typically driven by {@code topology.message.timeout.secs}.
+     */
     public WaitAckCache(String cacheSpec, Logger log, Consumer<Tuple> onEviction) {
         this(Caffeine.from(cacheSpec), log, onEviction);
     }
@@ -98,6 +99,7 @@ public class WaitAckCache {
                         .build();
     }
 
+    /** Returns the approximate number of entries in this cache. */
     public long estimatedSize() {
         return cache.estimatedSize();
     }
@@ -176,21 +178,27 @@ public class WaitAckCache {
             @Nullable ScopedCounter conflictCounter,
             TupleAction action) {
 
+        // opensearch-java: items() returns List<BulkResponseItem>; status() returns int
         var idsToBulkItems =
-                Arrays.stream(response.getItems())
+                response.items().stream()
                         .map(
                                 bir -> {
-                                    BulkItemResponse.Failure f = bir.getFailure();
+                                    var error = bir.error();
                                     boolean failed = false;
-                                    if (f != null) {
-                                        if (f.getStatus().equals(RestStatus.CONFLICT)) {
+                                    if (error != null) {
+                                        // opensearch-java: int status code, not RestStatus enum
+                                        if (bir.status() == 409) {
                                             if (conflictCounter != null) {
                                                 conflictCounter.scope("doc_conflicts").incrBy(1);
                                             }
-                                            log.debug("Doc conflict ID {}", bir.getId());
+                                            log.debug("Doc conflict ID {}", bir.id());
                                         } else {
                                             log.error(
-                                                    "Bulk item failure ID {}: {}", bir.getId(), f);
+                                                    "Bulk item failure ID {}: {}",
+                                                    bir.id(),
+                                                    error.reason() != null
+                                                            ? error.reason()
+                                                            : error.type());
                                             failed = true;
                                         }
                                     }
@@ -198,7 +206,9 @@ public class WaitAckCache {
                                 })
                         .collect(
                                 // https://github.com/apache/stormcrawler/issues/832
-                                Collectors.groupingBy(b -> b.id, Collectors.toUnmodifiableList()));
+                                Collectors.groupingBy(
+                                        BulkItemResponseToFailedFlag::id,
+                                        Collectors.toUnmodifiableList()));
 
         Map<String, List<Tuple>> presentTuples;
         long estimatedSize;
@@ -230,7 +240,7 @@ public class WaitAckCache {
             if (tuples != null) {
                 log.debug("Found {} tuple(s) for ID {}", tuples.size(), id);
                 for (Tuple t : tuples) {
-                    if (selected.failed) {
+                    if (selected.failed()) {
                         failureCount++;
                     } else {
                         ackCount++;
@@ -257,15 +267,22 @@ public class WaitAckCache {
         }
     }
 
-    /** Processes a failed bulk request by failing all associated tuples. */
+    /**
+     * Processes a failed bulk request by failing all associated tuples.
+     *
+     * @param failAction callback applied to each tuple that must be failed
+     */
     public void processFailedBulk(
             BulkRequest request, long executionId, Throwable failure, Consumer<Tuple> failAction) {
 
         log.error("Exception with bulk {} - failing the whole lot ", executionId, failure);
 
+        // opensearch-java: operations() + getBulkOperationId replaces
+        //   legacy requests() + DocWriteRequest::id
         final var failedIds =
-                request.requests().stream()
-                        .map(DocWriteRequest::id)
+                request.operations().stream()
+                        .map(OpenSearchConnection::getBulkOperationId)
+                        .filter(Objects::nonNull)
                         .collect(Collectors.toUnmodifiableSet());
 
         Map<String, List<Tuple>> failedTupleLists;
@@ -306,7 +323,7 @@ public class WaitAckCache {
         BulkItemResponseToFailedFlag best = items.get(0);
         int failedCount = 0;
         for (var item : items) {
-            if (item.failed) {
+            if (item.failed()) {
                 failedCount++;
             } else {
                 best = item;
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
index 16b3a005..f77662ea 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
@@ -28,13 +28,13 @@ import org.apache.storm.tuple.Tuple;
 import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
 import org.apache.stormcrawler.metrics.CrawlerMetrics;
+import org.apache.stormcrawler.opensearch.AsyncBulkProcessor;
 import org.apache.stormcrawler.opensearch.OpenSearchConnection;
 import org.apache.stormcrawler.opensearch.WaitAckCache;
 import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.action.bulk.BulkProcessor.Listener;
-import org.opensearch.action.bulk.BulkRequest;
-import org.opensearch.action.bulk.BulkResponse;
-import org.opensearch.action.delete.DeleteRequest;
+import org.opensearch.client.opensearch.core.BulkRequest;
+import org.opensearch.client.opensearch.core.BulkResponse;
+import org.opensearch.client.opensearch.core.bulk.BulkOperation;
 import org.slf4j.LoggerFactory;
 
 /**
@@ -43,7 +43,7 @@ import org.slf4j.LoggerFactory;
  * will also try to delete documents even though they were never indexed and it currently won't
  * delete documents which were indexed under the canonical URL.
  */
-public class DeletionBolt extends BaseRichBolt implements Listener {
+public class DeletionBolt extends BaseRichBolt implements AsyncBulkProcessor.Listener {
 
     static final org.slf4j.Logger LOG =
             LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -81,7 +81,7 @@ public class DeletionBolt extends BaseRichBolt implements Listener {
         try {
             connection = OpenSearchConnection.getConnection(conf, BOLT_TYPE, this);
         } catch (Exception e1) {
-            LOG.error("Can't connect to opensearch", e1);
+            LOG.error("Can't connect to OpenSearch", e1);
             throw new RuntimeException(e1);
         }
 
@@ -106,10 +106,12 @@ public class DeletionBolt extends BaseRichBolt implements Listener {
         // used
 
         final String docID = getDocumentID(metadata, url);
-        DeleteRequest dr = new DeleteRequest(getIndexName(metadata), docID);
-        connection.addToProcessor(dr);
+        final String targetIndex = getIndexName(metadata);
+        BulkOperation op = BulkOperation.of(b -> b.delete(d -> d.index(targetIndex).id(docID)));
 
         waitAck.addTuple(docID, tuple);
+
+        connection.addToProcessor(op);
     }
 
     @Override
@@ -155,9 +157,9 @@ public class DeletionBolt extends BaseRichBolt implements Listener {
         waitAck.processBulkResponse(
                 response,
                 executionId,
-                null, // no conflict counter — deletion conflicts are expected and only logged
+                null,
                 (id, t, selected) -> {
-                    if (!selected.failed) {
+                    if (!selected.failed()) {
                         _collector.ack(t);
                     } else {
                         String url = (String) t.getValueByField("url");
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
index 6191f2c3..f7a44185 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
@@ -18,9 +18,10 @@
 package org.apache.stormcrawler.opensearch.bolt;
 
 import static org.apache.stormcrawler.Constants.StatusStreamName;
-import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
 import org.apache.commons.lang3.StringUtils;
@@ -34,17 +35,15 @@ import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
 import org.apache.stormcrawler.metrics.CrawlerMetrics;
 import org.apache.stormcrawler.metrics.ScopedCounter;
 import org.apache.stormcrawler.metrics.ScopedReducedMetric;
+import org.apache.stormcrawler.opensearch.AsyncBulkProcessor;
 import org.apache.stormcrawler.opensearch.IndexCreation;
 import org.apache.stormcrawler.opensearch.OpenSearchConnection;
 import org.apache.stormcrawler.opensearch.WaitAckCache;
 import org.apache.stormcrawler.persistence.Status;
 import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.action.bulk.BulkProcessor;
-import org.opensearch.action.bulk.BulkRequest;
-import org.opensearch.action.bulk.BulkResponse;
-import org.opensearch.action.index.IndexRequest;
-import org.opensearch.core.rest.RestStatus;
-import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.client.opensearch.core.BulkRequest;
+import org.opensearch.client.opensearch.core.BulkResponse;
+import org.opensearch.client.opensearch.core.bulk.BulkOperation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -52,7 +51,7 @@ import org.slf4j.LoggerFactory;
  * Sends documents to opensearch. Indexes all the fields from the tuples or a Map
  * &lt;String,Object&gt; from a named field.
  */
-public class IndexerBolt extends AbstractIndexerBolt implements BulkProcessor.Listener {
+public class IndexerBolt extends AbstractIndexerBolt implements AsyncBulkProcessor.Listener {
 
     private static final Logger LOG = LoggerFactory.getLogger(IndexerBolt.class);
 
@@ -105,7 +104,7 @@ public class IndexerBolt extends AbstractIndexerBolt implements BulkProcessor.Li
         try {
             connection = OpenSearchConnection.getConnection(conf, OSBoltType, this);
         } catch (Exception e1) {
-            LOG.error("Can't connect to opensearch", e1);
+            LOG.error("Can't connect to OpenSearch", e1);
             throw new RuntimeException(e1);
         }
 
@@ -159,19 +158,19 @@ public class IndexerBolt extends AbstractIndexerBolt implements BulkProcessor.Li
         final String docID = getDocumentID(metadata, normalisedurl);
 
         try {
-            final XContentBuilder builder = jsonBuilder().startObject();
+            final Map<String, Object> source = new HashMap<>();
 
             // display text of the document?
             if (StringUtils.isNotBlank(fieldNameForText())) {
                 final String text = trimText(tuple.getStringByField("text"));
                 if (!ignoreEmptyFields() || StringUtils.isNotBlank(text)) {
-                    builder.field(fieldNameForText(), trimText(text));
+                    source.put(fieldNameForText(), trimText(text));
                 }
             }
 
             // send URL as field?
             if (StringUtils.isNotBlank(fieldNameForURL())) {
-                builder.field(fieldNameForURL(), normalisedurl);
+                source.put(fieldNameForURL(), normalisedurl);
             }
 
             // which metadata to display?
@@ -181,35 +180,54 @@ public class IndexerBolt extends AbstractIndexerBolt implements BulkProcessor.Li
                 if (entry.getValue().length == 1) {
                     final String value = entry.getValue()[0];
                     if (!ignoreEmptyFields() || StringUtils.isNotBlank(value)) {
-                        builder.field(entry.getKey(), value);
+                        source.put(entry.getKey(), value);
                     }
                 } else if (entry.getValue().length > 1) {
-                    builder.array(entry.getKey(), entry.getValue());
+                    source.put(entry.getKey(), Arrays.asList(entry.getValue()));
                 }
             }
 
-            builder.endObject();
-
-            final IndexRequest indexRequest =
-                    new IndexRequest(getIndexName(metadata))
-                            .source(builder)
-                            .id(docID)
-                            .create(create);
-
-            if (pipeline != null) {
-                indexRequest.setPipeline(pipeline);
+            final String targetIndex = getIndexName(metadata);
+            final BulkOperation op;
+            if (create) {
+                op =
+                        BulkOperation.of(
+                                b ->
+                                        b.create(
+                                                c -> {
+                                                    c.index(targetIndex).id(docID).document(source);
+                                                    if (pipeline != null) {
+                                                        c.pipeline(pipeline);
+                                                    }
+                                                    return c;
+                                                }));
+            } else {
+                op =
+                        BulkOperation.of(
+                                b ->
+                                        b.index(
+                                                idx -> {
+                                                    idx.index(targetIndex)
+                                                            .id(docID)
+                                                            .document(source);
+                                                    if (pipeline != null) {
+                                                        idx.pipeline(pipeline);
+                                                    }
+                                                    return idx;
+                                                }));
             }
 
-            connection.addToProcessor(indexRequest);
+            waitAck.addTuple(docID, tuple);
+
+            connection.addToProcessor(op);
 
             eventCounter.scope("Indexed").incrBy(1);
             perSecMetrics.scope("Indexed").update(1);
-
-            waitAck.addTuple(docID, tuple);
-        } catch (IOException e) {
+        } catch (Exception e) {
             LOG.error("Error building document for OpenSearch", e);
             // do not send to status stream so that it gets replayed
             _collector.fail(tuple);
+
             waitAck.invalidate(docID);
         }
     }
@@ -230,7 +248,7 @@ public class IndexerBolt extends AbstractIndexerBolt implements BulkProcessor.Li
     @Override
     public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
         eventCounter.scope("bulks_received").incrBy(1);
-        eventCounter.scope("bulk_msec").incrBy(response.getTook().getMillis());
+        eventCounter.scope("bulk_msec").incrBy(response.took());
 
         waitAck.processBulkResponse(
                 response,
@@ -240,7 +258,7 @@ public class IndexerBolt extends AbstractIndexerBolt implements BulkProcessor.Li
                     String url = (String) t.getValueByField("url");
                     Metadata metadata = (Metadata) t.getValueByField("metadata");
 
-                    if (!selected.failed) {
+                    if (!selected.failed()) {
                         _collector.emit(
                                 StatusStreamName, t, new Values(url, metadata, Status.FETCHED));
                         _collector.ack(t);
@@ -249,14 +267,16 @@ public class IndexerBolt extends AbstractIndexerBolt implements BulkProcessor.Li
                         LOG.error("update ID {}, URL {}, failure: {}", id, url, failure);
                         // there is something wrong with the content we should
                         // treat it as an ERROR
-                        if (failure.getStatus().equals(RestStatus.BAD_REQUEST)) {
+                        if (selected.getStatus() == 400) {
                             metadata.setValue(Constants.STATUS_ERROR_SOURCE, "OpenSearch indexing");
                             metadata.setValue(Constants.STATUS_ERROR_MESSAGE, "invalid content");
                             _collector.emit(
                                     StatusStreamName, t, new Values(url, metadata, Status.ERROR));
                             _collector.ack(t);
+                            LOG.debug("Acked {} with ID {}", url, id);
                         } else {
                             _collector.fail(t);
+                            LOG.debug("Failed {} with ID {}", url, id);
                         }
                     }
                 });
@@ -265,7 +285,6 @@ public class IndexerBolt extends AbstractIndexerBolt implements BulkProcessor.Li
     @Override
     public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
         eventCounter.scope("bulks_received").incrBy(1);
-
         waitAck.processFailedBulk(
                 request,
                 executionId,
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
index 345025fb..49ee7f0c 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
@@ -63,10 +63,7 @@ public class JSONURLFilterWrapper extends URLFilter {
     public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode filterParams) {
         refresher =
                 new DelegateRefresher<>(
-                        URLFilter.class,
-                        stormConf,
-                        filterParams,
-                        (delegate, conf, params) -> delegate.configure(conf, params));
+                        URLFilter.class, stormConf, filterParams, URLFilter::configure);
     }
 
     @Override
@@ -79,6 +76,8 @@ public class JSONURLFilterWrapper extends URLFilter {
 
     @Override
     public void cleanup() {
-        refresher.cleanup();
+        if (refresher != null) {
+            refresher.cleanup();
+        }
     }
 }
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
index 6b9ccf4c..c46a2b73 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
@@ -17,12 +17,11 @@
 
 package org.apache.stormcrawler.opensearch.metrics;
 
-import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
-
 import java.io.IOException;
 import java.text.SimpleDateFormat;
 import java.util.Collection;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -32,8 +31,7 @@ import org.apache.storm.task.TopologyContext;
 import org.apache.stormcrawler.opensearch.IndexCreation;
 import org.apache.stormcrawler.opensearch.OpenSearchConnection;
 import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.action.index.IndexRequest;
-import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.client.opensearch.core.bulk.BulkOperation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -144,19 +142,20 @@ public class MetricsConsumer implements IMetricsConsumer {
 
     private void indexDataPoint(TaskInfo taskInfo, Date timestamp, String name, double value) {
         try {
-            XContentBuilder builder = jsonBuilder().startObject();
-            builder.field("stormId", stormID);
-            builder.field("srcComponentId", taskInfo.srcComponentId);
-            builder.field("srcTaskId", taskInfo.srcTaskId);
-            builder.field("srcWorkerHost", taskInfo.srcWorkerHost);
-            builder.field("srcWorkerPort", taskInfo.srcWorkerPort);
-            builder.field("name", name);
-            builder.field("value", value);
-            builder.field("timestamp", timestamp);
-            builder.endObject();
-
-            IndexRequest indexRequest = new IndexRequest(getIndexName(timestamp)).source(builder);
-            connection.addToProcessor(indexRequest);
+            Map<String, Object> doc = new HashMap<>();
+            doc.put("stormId", stormID);
+            doc.put("srcComponentId", taskInfo.srcComponentId);
+            doc.put("srcTaskId", taskInfo.srcTaskId);
+            doc.put("srcWorkerHost", taskInfo.srcWorkerHost);
+            doc.put("srcWorkerPort", taskInfo.srcWorkerPort);
+            doc.put("name", name);
+            doc.put("value", value);
+            doc.put("timestamp", timestamp.toInstant().toString());
+
+            final String targetIndex = getIndexName(timestamp);
+            BulkOperation op =
+                    BulkOperation.of(b -> b.index(idx -> idx.index(targetIndex).document(doc)));
+            connection.addToProcessor(op);
         } catch (Exception e) {
             LOG.error("problem when building request for OpenSearch", e);
         }
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java
index 3957ba6a..d3b654a0 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java
@@ -17,8 +17,6 @@
 
 package org.apache.stormcrawler.opensearch.metrics;
 
-import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
-
 import com.codahale.metrics.Counter;
 import com.codahale.metrics.Gauge;
 import com.codahale.metrics.Histogram;
@@ -30,6 +28,7 @@ import com.codahale.metrics.Timer;
 import java.io.IOException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
 import java.util.SortedMap;
@@ -38,8 +37,7 @@ import org.apache.storm.metrics2.reporters.ScheduledStormReporter;
 import org.apache.stormcrawler.opensearch.IndexCreation;
 import org.apache.stormcrawler.opensearch.OpenSearchConnection;
 import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.action.index.IndexRequest;
-import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.client.opensearch.core.bulk.BulkOperation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -68,6 +66,8 @@ public class MetricsReporter extends ScheduledStormReporter {
 
     private static final String DATE_FORMAT_KEY = "opensearch.metrics.date.format";
 
+    private ScheduledReporter reporter;
+
     @Override
     public void prepare(
             MetricRegistry metricsRegistry,
@@ -107,8 +107,6 @@ public class MetricsReporter extends ScheduledStormReporter {
         reporter.start(reportPeriod, reportPeriodUnit);
     }
 
-    private ScheduledReporter reporter;
-
     @Override
     public void start() {
         // already started in prepare()
@@ -123,7 +121,7 @@ public class MetricsReporter extends ScheduledStormReporter {
 
     /**
      * Inner ScheduledReporter that writes Codahale metrics to OpenSearch in the same format as the
-     * V1 MetricsConsumer.
+     * V1 {@link MetricsConsumer}.
      */
     private static class OpenSearchScheduledReporter extends ScheduledReporter {
 
@@ -203,16 +201,16 @@ public class MetricsReporter extends ScheduledStormReporter {
 
         private void indexDataPoint(Date timestamp, String name, double value) {
             try {
-                XContentBuilder builder = jsonBuilder().startObject();
-                builder.field("stormId", stormId);
-                builder.field("name", name);
-                builder.field("value", value);
-                builder.field("timestamp", timestamp);
-                builder.endObject();
-
-                IndexRequest indexRequest =
-                        new IndexRequest(getIndexName(timestamp)).source(builder);
-                connection.addToProcessor(indexRequest);
+                Map<String, Object> doc = new HashMap<>();
+                doc.put("stormId", stormId);
+                doc.put("name", name);
+                doc.put("value", value);
+                doc.put("timestamp", timestamp.toInstant().toString());
+
+                final String targetIndex = getIndexName(timestamp);
+                BulkOperation op =
+                        BulkOperation.of(b -> b.index(idx -> idx.index(targetIndex).document(doc)));
+                connection.addToProcessor(op);
             } catch (Exception e) {
                 LOG.error("Problem when building request for OpenSearch", e);
             }
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
index 0b71e101..c74184c2 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
@@ -17,8 +17,11 @@
 
 package org.apache.stormcrawler.opensearch.metrics;
 
-import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.storm.Config;
 import org.apache.storm.task.OutputCollector;
 import org.apache.storm.task.TopologyContext;
@@ -30,12 +33,8 @@ import org.apache.stormcrawler.metrics.CrawlerMetrics;
 import org.apache.stormcrawler.opensearch.Constants;
 import org.apache.stormcrawler.opensearch.OpenSearchConnection;
 import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.RequestOptions;
-import org.opensearch.client.core.CountRequest;
-import org.opensearch.client.core.CountResponse;
-import org.opensearch.core.action.ActionListener;
-import org.opensearch.index.query.QueryBuilders;
-import org.opensearch.search.builder.SearchSourceBuilder;
+import org.opensearch.client.opensearch.OpenSearchClient;
+import org.opensearch.client.opensearch._types.FieldValue;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -53,45 +52,22 @@ public class StatusMetricsBolt extends BaseRichBolt {
 
     private String indexName;
 
-    private OpenSearchConnection connection;
+    private OpenSearchClient client;
 
-    private Map<String, Long> latestStatusCounts = new HashMap<>(6);
+    private Map<String, Long> latestStatusCounts = new ConcurrentHashMap<>(6);
 
     private int freqStats = 60;
 
     private OutputCollector _collector;
 
-    private transient StatusActionListener[] listeners;
+    private transient StatusCounter[] counters;
 
-    private class StatusActionListener implements ActionListener<CountResponse> {
+    private static final class StatusCounter {
+        final String name;
+        final AtomicBoolean ready = new AtomicBoolean(true);
 
-        private final String name;
-
-        private boolean ready = true;
-
-        public boolean isReady() {
-            return ready;
-        }
-
-        public void busy() {
-            this.ready = false;
-        }
-
-        StatusActionListener(String statusName) {
-            name = statusName;
-        }
-
-        @Override
-        public void onResponse(CountResponse response) {
-            ready = true;
-            LOG.debug("Got {} counts for status:{}", response.getCount(), name);
-            latestStatusCounts.put(name, response.getCount());
-        }
-
-        @Override
-        public void onFailure(Exception e) {
-            ready = true;
-            LOG.error("Failure when getting counts for status:{}", name, e);
+        StatusCounter(String name) {
+            this.name = name;
         }
     }
 
@@ -101,7 +77,7 @@ public class StatusMetricsBolt extends BaseRichBolt {
         _collector = collector;
         indexName = ConfUtils.getString(stormConf, OSStatusIndexNameParamName, "status");
         try {
-            connection = OpenSearchConnection.getConnection(stormConf, OSBoltType);
+            client = OpenSearchConnection.getClient(stormConf, OSBoltType);
         } catch (Exception e1) {
             LOG.error("Can't connect to OpenSearch", e1);
             throw new RuntimeException(e1);
@@ -110,14 +86,14 @@ public class StatusMetricsBolt extends BaseRichBolt {
         CrawlerMetrics.registerGauge(
                 context, stormConf, "status.count", () -> latestStatusCounts, freqStats);
 
-        listeners = new StatusActionListener[6];
+        counters = new StatusCounter[6];
 
-        listeners[0] = new StatusActionListener("DISCOVERED");
-        listeners[1] = new StatusActionListener("FETCHED");
-        listeners[2] = new StatusActionListener("FETCH_ERROR");
-        listeners[3] = new StatusActionListener("REDIRECTION");
-        listeners[4] = new StatusActionListener("ERROR");
-        listeners[5] = new StatusActionListener("TOTAL");
+        counters[0] = new StatusCounter("DISCOVERED");
+        counters[1] = new StatusCounter("FETCHED");
+        counters[2] = new StatusCounter("FETCH_ERROR");
+        counters[3] = new StatusCounter("REDIRECTION");
+        counters[4] = new StatusCounter("ERROR");
+        counters[5] = new StatusCounter("TOTAL");
     }
 
     @Override
@@ -137,26 +113,69 @@ public class StatusMetricsBolt extends BaseRichBolt {
             return;
         }
 
-        for (StatusActionListener listener : listeners) {
+        for (StatusCounter counter : counters) {
             // still waiting for results from previous request
-            if (!listener.isReady()) {
-                LOG.debug("Not ready to get counts for status {}", listener.name);
+            if (!counter.ready.compareAndSet(true, false)) {
+                LOG.debug("Not ready to get counts for status {}", counter.name);
                 continue;
             }
-            CountRequest request = new CountRequest(indexName);
-            if (!listener.name.equalsIgnoreCase("TOTAL")) {
-                SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-                sourceBuilder.query(QueryBuilders.termQuery("status", listener.name));
-                request.source(sourceBuilder);
-            }
-            listener.busy();
-            connection.getClient().countAsync(request, RequestOptions.DEFAULT, listener);
+            final String statusName = counter.name;
+            CompletableFuture.supplyAsync(
+                            () -> {
+                                try {
+                                    if (statusName.equalsIgnoreCase("TOTAL")) {
+                                        return client.count(c -> c.index(indexName));
+                                    } else {
+                                        return client.count(
+                                                c ->
+                                                        c.index(indexName)
+                                                                .query(
+                                                                        q ->
+                                                                                q.term(
+                                                                                        t ->
+                                                                                                t.field(
+                                                                                                                "status")
+                                                                                                        .value(
+                                                                                                                FieldValue
+                                                                                                                        .of(
+                                                                                                                                statusName)))));
+                                    }
+                                } catch (Exception e) {
+                                    throw new CompletionException(e);
+                                }
+                            })
+                    .thenAccept(
+                            response -> {
+                                counter.ready.set(true);
+                                LOG.debug(
+                                        "Got {} counts for status:{}",
+                                        response.count(),
+                                        statusName);
+                                latestStatusCounts.put(statusName, response.count());
+                            })
+                    .exceptionally(
+                            e -> {
+                                counter.ready.set(true);
+                                Throwable cause =
+                                        e instanceof CompletionException ? e.getCause() : e;
+                                LOG.error(
+                                        "Failure when getting counts for status:{}",
+                                        statusName,
+                                        cause);
+                                return null;
+                            });
         }
     }
 
     @Override
     public void cleanup() {
-        connection.close();
+        if (client != null) {
+            try {
+                client._transport().close();
+            } catch (Exception e) {
+                LOG.error("Exception closing client transport", e);
+            }
+        }
     }
 
     @Override
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
index d8995ff8..6f6d7be6 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
@@ -62,10 +62,7 @@ public class JSONResourceWrapper extends ParseFilter {
     public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode filterParams) {
         refresher =
                 new DelegateRefresher<>(
-                        ParseFilter.class,
-                        stormConf,
-                        filterParams,
-                        (delegate, conf, params) -> delegate.configure(conf, params));
+                        ParseFilter.class, stormConf, filterParams, ParseFilter::configure);
     }
 
     @Override
@@ -75,6 +72,8 @@ public class JSONResourceWrapper extends ParseFilter {
 
     @Override
     public void cleanup() {
-        refresher.cleanup();
+        if (refresher != null) {
+            refresher.cleanup();
+        }
     }
 }
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
index b29ebd16..0afe2b19 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
@@ -19,10 +19,15 @@ package org.apache.stormcrawler.opensearch.persistence;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Date;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import org.apache.storm.spout.SpoutOutputCollector;
 import org.apache.storm.task.TopologyContext;
 import org.apache.stormcrawler.Metadata;
@@ -31,8 +36,8 @@ import org.apache.stormcrawler.opensearch.IndexCreation;
 import org.apache.stormcrawler.opensearch.OpenSearchConnection;
 import org.apache.stormcrawler.persistence.AbstractQueryingSpout;
 import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.RestHighLevelClient;
-import org.opensearch.search.SearchHit;
+import org.opensearch.client.json.JsonData;
+import org.opensearch.client.opensearch.OpenSearchClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -72,7 +77,7 @@ public abstract class AbstractSpout extends AbstractQueryingSpout {
 
     protected String indexName;
 
-    protected static RestHighLevelClient client;
+    protected static OpenSearchClient client;
 
     /**
      * when using multiple instances - each one is in charge of a specific shard useful when
@@ -98,6 +103,15 @@ public abstract class AbstractSpout extends AbstractQueryingSpout {
 
     protected int queryTimeout = -1;
 
+    /**
+     * Single-threaded executor dedicated to running the asynchronous OpenSearch queries triggered
+     * from this spout. Using a dedicated, named, daemon executor avoids polluting the JVM-wide
+     * {@link java.util.concurrent.ForkJoinPool#commonPool() common pool} (which is shared with all
+     * other Storm tasks running in the same worker) and allows clean shutdown when the topology is
+     * stopped.
+     */
+    protected ExecutorService queryExecutor;
+
     @Override
     public void open(
             Map<String, Object> stormConf,
@@ -106,6 +120,19 @@ public abstract class AbstractSpout extends AbstractQueryingSpout {
 
         super.open(stormConf, context, collector);
 
+        final String executorName =
+                "OpenSearchSpout-"
+                        + context.getThisComponentId()
+                        + "-"
+                        + context.getThisTaskIndex();
+        queryExecutor =
+                Executors.newSingleThreadExecutor(
+                        r -> {
+                            Thread t = new Thread(r, executorName);
+                            t.setDaemon(true);
+                            return t;
+                        });
+
         indexName = ConfUtils.getString(stormConf, OSStatusIndexNameParamName, "status");
 
         // one OS client per JVM
@@ -180,14 +207,59 @@ public abstract class AbstractSpout extends AbstractQueryingSpout {
     /** Builds a query and use it retrieve the results from OS * */
     protected abstract void populateBuffer();
 
-    protected final boolean addHitToBuffer(SearchHit hit) {
-        Map<String, Object> keyValues = hit.getSourceAsMap();
-        String url = (String) keyValues.get("url");
+    /**
+     * Converts the {@link JsonData} carried by a search hit into a plain {@code Map<String,
+     * Object>} populated with native Java types (strings, numbers, lists, nested maps).
+     *
+     * <p>This helper exists to work around a subtle short-circuit inside {@code JsonDataImpl#to}:
+     *
+     * <pre>{@code
+     * if (clazz.isAssignableFrom(value.getClass())) {
+     *     return (T) value;
+     * }
+     * }</pre>
+     *
+     * <p>When {@link JsonData} is produced via {@code JsonData._DESERIALIZER} (as opposed to a raw
+     * Jackson {@code TreeNode}), its internal {@code value} is a Jakarta JSON-P {@code JsonObject}
+     * — typically a Parsson {@code JsonObjectImpl}. Crucially, {@code jakarta.json.JsonObject
+     * extends Map<String, JsonValue>}, so calling {@code source.to(Map.class)} or {@code
+     * source.to(Object.class)} matches the {@code isAssignableFrom} guard and returns the raw
+     * {@code JsonObject} <em>without ever consulting the configured {@link
+     * org.opensearch.client.json.jackson.JacksonJsonpMapper}</em>. The resulting map values are
+     * still {@code JsonString}/{@code JsonStringImpl} instances and any downstream {@code (String)
+     * value} cast blows up with a {@link ClassCastException}.
+     *
+     * <p>Passing a concrete subclass that {@code JsonObject} does <em>not</em> implement (here
+     * {@link LinkedHashMap}) bypasses the short-circuit and forces the round-trip through {@code
+     * JacksonJsonpGenerator -> JacksonJsonpParser -> Jackson ObjectMapper.readValue}, which yields
+     * native Java values (String, Long, Double, ArrayList, LinkedHashMap…).
+     *
+     * @param source the document source carried by an OpenSearch hit; may be {@code null}
+     * @return a (possibly empty) map of native Java values, never {@code null}
+     */
+    static Map<String, Object> sourceAsMap(JsonData source) {
+        if (source == null) {
+            return Collections.emptyMap();
+        }
+        @SuppressWarnings("unchecked")
+        Map<String, Object> map = source.to(LinkedHashMap.class);
+        return map == null ? Collections.emptyMap() : map;
+    }
+
+    /**
+     * Adds a document source to the URL buffer unless it is already being processed.
+     *
+     * @param source the document source as a key-value map (must contain a "url" entry)
+     * @return {@code true} if the URL was added to the buffer, {@code false} if it was already
+     *     being processed or already present
+     */
+    protected final boolean addHitToBuffer(Map<String, Object> source) {
+        String url = (String) source.get("url");
         // is already being processed - skip it!
         if (beingProcessed.containsKey(url)) {
             return false;
         }
-        return buffer.add(url, fromKeyValues(keyValues));
+        return buffer.add(url, fromKeyValues(source));
     }
 
     protected final Metadata fromKeyValues(Map<String, Object> keyValues) {
@@ -225,10 +297,29 @@ public abstract class AbstractSpout extends AbstractQueryingSpout {
 
     @Override
     public void close() {
+        // Shut the dedicated query executor down first so no more requests can be enqueued
+        // against the client while we are tearing it down. We give in-flight queries a short
+        // grace period to drain to keep shutdown reasonably graceful.
+        if (queryExecutor != null) {
+            queryExecutor.shutdown();
+            try {
+                if (!queryExecutor.awaitTermination(30, TimeUnit.SECONDS)) {
+                    LOG.warn(
+                            "{} Query executor did not terminate within 30s, forcing shutdown",
+                            logIdprefix);
+                    queryExecutor.shutdownNow();
+                }
+            } catch (InterruptedException e) {
+                queryExecutor.shutdownNow();
+                Thread.currentThread().interrupt();
+            }
+            queryExecutor = null;
+        }
+
         synchronized (AbstractSpout.class) {
             if (client != null) {
                 try {
-                    client.close();
+                    client._transport().close();
                 } catch (IOException e) {
                     LOG.error("Exception caught when closing client", e);
                 }
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java
index 79936b36..a6272f36 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java
@@ -17,45 +17,38 @@
 
 package org.apache.stormcrawler.opensearch.persistence;
 
-import static org.opensearch.index.query.QueryBuilders.boolQuery;
-
+import java.io.IOException;
 import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
 import java.util.Calendar;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.TimeZone;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.storm.spout.SpoutOutputCollector;
 import org.apache.storm.task.TopologyContext;
 import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.opensearch.Constants;
 import org.apache.stormcrawler.util.ConfUtils;
-import org.joda.time.format.ISODateTimeFormat;
-import org.opensearch.action.search.SearchRequest;
-import org.opensearch.action.search.SearchResponse;
-import org.opensearch.client.RequestOptions;
-import org.opensearch.core.action.ActionListener;
-import org.opensearch.index.query.BoolQueryBuilder;
-import org.opensearch.index.query.QueryBuilders;
-import org.opensearch.search.SearchHit;
-import org.opensearch.search.aggregations.AggregationBuilders;
-import org.opensearch.search.aggregations.Aggregations;
-import org.opensearch.search.aggregations.BucketOrder;
-import org.opensearch.search.aggregations.bucket.SingleBucketAggregation;
-import org.opensearch.search.aggregations.bucket.sampler.DiversifiedAggregationBuilder;
-import org.opensearch.search.aggregations.bucket.terms.Terms;
-import org.opensearch.search.aggregations.bucket.terms.Terms.Bucket;
-import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
-import org.opensearch.search.aggregations.metrics.TopHits;
-import org.opensearch.search.builder.SearchSourceBuilder;
-import org.opensearch.search.sort.FieldSortBuilder;
-import org.opensearch.search.sort.SortBuilders;
-import org.opensearch.search.sort.SortOrder;
+import org.opensearch.client.json.JsonData;
+import org.opensearch.client.opensearch._types.FieldValue;
+import org.opensearch.client.opensearch._types.SortOrder;
+import org.opensearch.client.opensearch._types.aggregations.Aggregate;
+import org.opensearch.client.opensearch._types.aggregations.Aggregation;
+import org.opensearch.client.opensearch._types.aggregations.StringTermsBucket;
+import org.opensearch.client.opensearch._types.aggregations.TopHitsAggregate;
+import org.opensearch.client.opensearch.core.SearchRequest;
+import org.opensearch.client.opensearch.core.SearchResponse;
+import org.opensearch.client.opensearch.core.search.Hit;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -65,7 +58,7 @@ import org.slf4j.LoggerFactory;
  * the same number of spout instances as OpenSearch shards. Guarantees a good mix of URLs by
  * aggregating them by an arbitrary field e.g. key.
  */
-public class AggregationSpout extends AbstractSpout implements ActionListener<SearchResponse> {
+public class AggregationSpout extends AbstractSpout {
 
     private static final Logger LOG = LoggerFactory.getLogger(AggregationSpout.class);
 
@@ -104,106 +97,171 @@ public class AggregationSpout extends AbstractSpout implements ActionListener<Se
             lastTimeResetToNow = Instant.now();
         }
 
-        String formattedQueryDate = ISODateTimeFormat.dateTimeNoMillis().print(queryDate.getTime());
+        String formattedQueryDate =
+                Instant.ofEpochMilli(queryDate.getTime())
+                        .atOffset(ZoneOffset.UTC)
+                        .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
 
         LOG.info("{} Populating buffer with nextFetchDate <= {}", logIdprefix, formattedQueryDate);
 
-        BoolQueryBuilder queryBuilder =
-                boolQuery()
-                        .filter(QueryBuilders.rangeQuery("nextFetchDate").lte(formattedQueryDate));
-
-        if (filterQueries != null) {
-            for (String filterQuery : filterQueries) {
-                queryBuilder.filter(QueryBuilders.queryStringQuery(filterQuery));
-            }
+        // Build the top_hits sub-aggregation
+        Aggregation topHitsAgg =
+                Aggregation.of(
+                        a ->
+                                a.topHits(
+                                        th -> {
+                                            th.size(maxURLsPerBucket).explain(false);
+                                            for (String bsf : bucketSortField) {
+                                                th.sort(
+                                                        s ->
+                                                                s.field(
+                                                                        fs ->
+                                                                                fs.field(bsf)
+                                                                                        .order(
+                                                                                                SortOrder
+                                                                                                        .Asc)));
+                                            }
+                                            return th;
+                                        }));
+
+        // Build the terms (partition) aggregation with top_hits sub-agg
+        Aggregation.Builder.ContainerBuilder partitionAggBuilder =
+                new Aggregation.Builder()
+                        .terms(
+                                t -> {
+                                    t.field(partitionField).size(maxBucketNum);
+                                    // sort between buckets by the min sub-aggregation
+                                    if (StringUtils.isNotBlank(totalSortField)) {
+                                        t.order(
+                                                Collections.singletonList(
+                                                        Collections.singletonMap(
+                                                                "top_hit", SortOrder.Asc)));
+                                    }
+                                    return t;
+                                })
+                        .aggregations("docs", topHitsAgg);
+
+        // add the min sub-aggregation used for sorting between buckets
+        if (StringUtils.isNotBlank(totalSortField)) {
+            partitionAggBuilder.aggregations(
+                    "top_hit", Aggregation.of(minAgg -> minAgg.min(m -> m.field(totalSortField))));
         }
 
-        SearchRequest request = new SearchRequest(indexName);
-
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-        sourceBuilder.query(queryBuilder);
-        sourceBuilder.from(0);
-        sourceBuilder.size(0);
-        sourceBuilder.explain(false);
-        sourceBuilder.trackTotalHits(false);
+        Aggregation partitionAgg = partitionAggBuilder.build();
+
+        // Build the search request
+        SearchRequest.Builder requestBuilder =
+                new SearchRequest.Builder()
+                        .index(indexName)
+                        .size(0)
+                        .trackTotalHits(t -> t.enabled(false))
+                        .query(
+                                q ->
+                                        q.bool(
+                                                b -> {
+                                                    b.filter(
+                                                            f ->
+                                                                    f.range(
+                                                                            r ->
+                                                                                    r.field(
+                                                                                                    "nextFetchDate")
+                                                                                            .lte(
+                                                                                                    JsonData
+                                                                                                            .of(
+                                                                                                                    formattedQueryDate))));
+                                                    if (filterQueries != null) {
+                                                        for (String fq : filterQueries) {
+                                                            b.filter(
+                                                                    f ->
+                                                                            f.queryString(
+                                                                                    qs ->
+                                                                                            qs
+                                                                                                    .query(
+                                                                                                            fq)));
+                                                        }
+                                                    }
+                                                    return b;
+                                                }));
 
         if (queryTimeout != -1) {
-            sourceBuilder.timeout(
-                    new org.opensearch.common.unit.TimeValue(queryTimeout, TimeUnit.SECONDS));
-        }
-
-        TermsAggregationBuilder aggregations =
-                AggregationBuilders.terms("partition").field(partitionField).size(maxBucketNum);
-
-        org.opensearch.search.aggregations.metrics.TopHitsAggregationBuilder tophits =
-                AggregationBuilders.topHits("docs").size(maxURLsPerBucket).explain(false);
-
-        // sort within a bucket
-        for (String bsf : bucketSortField) {
-            FieldSortBuilder sorter = SortBuilders.fieldSort(bsf).order(SortOrder.ASC);
-            tophits.sort(sorter);
-        }
-
-        aggregations.subAggregation(tophits);
-
-        // sort between buckets
-        if (StringUtils.isNotBlank(totalSortField)) {
-            org.opensearch.search.aggregations.metrics.MinAggregationBuilder minBuilder =
-                    AggregationBuilders.min("top_hit").field(totalSortField);
-            aggregations.subAggregation(minBuilder);
-            aggregations.order(BucketOrder.aggregation("top_hit", true));
+            requestBuilder.timeout(queryTimeout + "s");
         }
 
         if (sample) {
-            DiversifiedAggregationBuilder sab = new DiversifiedAggregationBuilder("sample");
-            sab.field(partitionField).maxDocsPerValue(maxURLsPerBucket);
-            sab.shardSize(maxURLsPerBucket * maxBucketNum);
-            sab.subAggregation(aggregations);
-            sourceBuilder.aggregation(sab);
+            // Wrap in a diversified sampler aggregation
+            requestBuilder.aggregations(
+                    "sample",
+                    Aggregation.of(
+                            a ->
+                                    a.diversifiedSampler(
+                                                    ds ->
+                                                            ds.field(partitionField)
+                                                                    .maxDocsPerValue(
+                                                                            maxURLsPerBucket)
+                                                                    .shardSize(
+                                                                            maxURLsPerBucket
+                                                                                    * maxBucketNum))
+                                            .aggregations("partition", partitionAgg)));
         } else {
-            sourceBuilder.aggregation(aggregations);
+            requestBuilder.aggregations("partition", partitionAgg);
         }
 
-        request.source(sourceBuilder);
-
-        // https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-preference.html
-        // _shards:2,3
-        // specific shard but ideally a local copy of it
+        // shard preference for routing
         if (shardID != -1) {
-            request.preference("_shards:" + shardID + "|_local");
+            requestBuilder.preference("_shards:" + shardID + "|_local");
         }
 
+        SearchRequest request = requestBuilder.build();
+
         // dump query to log
         LOG.debug("{} OpenSearch query {}", logIdprefix, request);
 
         LOG.trace("{} isInQuery set to true", logIdprefix);
         isInQuery.set(true);
-        client.searchAsync(request, RequestOptions.DEFAULT, this);
-    }
 
-    @Override
-    public void onFailure(Exception arg0) {
-        LOG.error("{} Exception with OpenSearch query", logIdprefix, arg0);
-        markQueryReceivedNow();
+        CompletableFuture.supplyAsync(
+                        () -> {
+                            try {
+                                return client.search(request, JsonData.class);
+                            } catch (IOException e) {
+                                throw new CompletionException(e);
+                            }
+                        },
+                        queryExecutor)
+                .thenAccept(this::handleResponse)
+                .exceptionally(
+                        e -> {
+                            Throwable cause = e instanceof CompletionException ? e.getCause() : e;
+                            LOG.error("{} Exception with OpenSearch query", logIdprefix, cause);
+                            markQueryReceivedNow();
+                            return null;
+                        });
     }
 
-    @Override
-    public void onResponse(SearchResponse response) {
+    /**
+     * Handles the search response from an asynchronous aggregation query, extracting URLs from term
+     * buckets and adding them to the buffer.
+     *
+     * @param response the search response containing aggregation results
+     */
+    protected void handleResponse(SearchResponse<JsonData> response) {
         long timeTaken = System.currentTimeMillis() - getTimeLastQuerySent();
 
-        Aggregations aggregs = response.getAggregations();
+        Map<String, Aggregate> aggregs = response.aggregations();
 
-        if (aggregs == null) {
+        if (aggregs == null || aggregs.isEmpty()) {
             markQueryReceivedNow();
             return;
         }
 
-        SingleBucketAggregation sample = aggregs.get("sample");
-        if (sample != null) {
-            aggregs = sample.getAggregations();
+        // Unwrap the sample aggregation if present
+        Aggregate sampleAgg = aggregs.get("sample");
+        if (sampleAgg != null) {
+            aggregs = sampleAgg.sampler().aggregations();
         }
 
-        Terms agg = aggregs.get("partition");
+        Aggregate partitionAgg = aggregs.get("partition");
+        List<StringTermsBucket> buckets = partitionAgg.sterms().buckets().array();
 
         int numhits = 0;
         int numBuckets = 0;
@@ -214,35 +272,31 @@ public class AggregationSpout extends AbstractSpout implements ActionListener<Se
         currentBuckets.clear();
 
         // For each entry
-        Iterator<Terms.Bucket> iterator = (Iterator<Bucket>) agg.getBuckets().iterator();
+        Iterator<StringTermsBucket> iterator = buckets.iterator();
         while (iterator.hasNext()) {
-            Terms.Bucket entry = iterator.next();
-            String key = (String) entry.getKey(); // bucket key
+            StringTermsBucket entry = iterator.next();
+            String key = entry.key(); // bucket key
 
             currentBuckets.add(key);
 
-            long docCount = entry.getDocCount(); // Doc count
+            long docCount = entry.docCount(); // Doc count
 
             int hitsForThisBucket = 0;
 
-            SearchHit lastHit = null;
-
+            List<FieldValue> lastSortValues = null;
             // filter results so that we don't include URLs we are already
             // being processed
-            TopHits topHits = entry.getAggregations().get("docs");
-            for (SearchHit hit : topHits.getHits().getHits()) {
+            TopHitsAggregate topHits = entry.aggregations().get("docs").topHits();
+            for (Hit<JsonData> hit : topHits.hits().hits()) {
 
-                LOG.debug(
-                        "{} -> id [{}], _source [{}]",
-                        logIdprefix,
-                        hit.getId(),
-                        hit.getSourceAsString());
+                Map<String, Object> keyValues = sourceAsMap(hit.source());
+
+                LOG.debug("{} -> id [{}], _source [{}]", logIdprefix, hit.id(), keyValues);
 
                 hitsForThisBucket++;
 
-                lastHit = hit;
+                lastSortValues = hit.sort();
 
-                Map<String, Object> keyValues = hit.getSourceAsMap();
                 String url = (String) keyValues.get("url");
 
                 // consider only the first document of the last bucket
@@ -273,8 +327,8 @@ public class AggregationSpout extends AbstractSpout implements ActionListener<Se
                 LOG.debug("{} -> added to buffer : {}", logIdprefix, url);
             }
 
-            if (lastHit != null) {
-                sortValuesForKey(key, lastHit.getSortValues());
+            if (lastSortValues != null && !lastSortValues.isEmpty()) {
+                sortValuesForKey(key, lastSortValues.toArray());
             }
 
             if (hitsForThisBucket > 0) {
@@ -369,5 +423,10 @@ public class AggregationSpout extends AbstractSpout implements ActionListener<Se
         markQueryReceivedNow();
     }
 
+    /**
+     * Receives the sort values of the last hit returned for a bucket, as the {@link FieldValue}
+     * instances of the client. Subclasses can pass them to {@code search_after} to read the bucket
+     * from that position.
+     */
     protected void sortValuesForKey(String key, Object[] sortValues) {}
 }
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
index 551153f5..b3301e5c 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
@@ -17,31 +17,28 @@
 
 package org.apache.stormcrawler.opensearch.persistence;
 
-import static org.opensearch.index.query.QueryBuilders.boolQuery;
-
 import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.Caffeine;
+import java.io.IOException;
 import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
 import org.apache.storm.spout.SpoutOutputCollector;
 import org.apache.storm.task.TopologyContext;
 import org.apache.stormcrawler.opensearch.Constants;
 import org.apache.stormcrawler.persistence.EmptyQueueListener;
 import org.apache.stormcrawler.util.ConfUtils;
-import org.joda.time.format.ISODateTimeFormat;
-import org.opensearch.action.search.SearchRequest;
-import org.opensearch.action.search.SearchResponse;
-import org.opensearch.client.RequestOptions;
-import org.opensearch.core.action.ActionListener;
-import org.opensearch.index.query.BoolQueryBuilder;
-import org.opensearch.index.query.QueryBuilders;
-import org.opensearch.search.SearchHit;
-import org.opensearch.search.builder.SearchSourceBuilder;
-import org.opensearch.search.sort.FieldSortBuilder;
-import org.opensearch.search.sort.SortBuilders;
-import org.opensearch.search.sort.SortOrder;
+import org.opensearch.client.json.JsonData;
+import org.opensearch.client.opensearch._types.FieldValue;
+import org.opensearch.client.opensearch._types.SortOrder;
+import org.opensearch.client.opensearch.core.SearchRequest;
+import org.opensearch.client.opensearch.core.SearchResponse;
+import org.opensearch.client.opensearch.core.search.Hit;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -90,7 +87,7 @@ public class HybridSpout extends AggregationSpout implements EmptyQueueListener
         // would just overload OpenSearch and yield
         // mainly duplicates
         if (isInQuery.get()) {
-            LOG.trace("{} isInquery true", logIdprefix, queueName);
+            LOG.trace("{} isInquery true for {}", logIdprefix, queueName);
             return;
         }
 
@@ -101,57 +98,89 @@ public class HybridSpout extends AggregationSpout implements EmptyQueueListener
             lastTimeResetToNow = Instant.now();
         }
 
-        String formattedQueryDate = ISODateTimeFormat.dateTimeNoMillis().print(queryDate.getTime());
-
-        BoolQueryBuilder queryBuilder =
-                boolQuery()
-                        .filter(QueryBuilders.rangeQuery("nextFetchDate").lte(formattedQueryDate));
-
-        queryBuilder.filter(QueryBuilders.termQuery(partitionField, queueName));
-
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-        sourceBuilder.query(queryBuilder);
-        sourceBuilder.from(0);
-        sourceBuilder.size(bufferReloadSize);
-        sourceBuilder.explain(false);
-        sourceBuilder.trackTotalHits(false);
+        String formattedQueryDate =
+                Instant.ofEpochMilli(queryDate.getTime())
+                        .atOffset(ZoneOffset.UTC)
+                        .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
+
+        SearchRequest.Builder requestBuilder =
+                new SearchRequest.Builder()
+                        .index(indexName)
+                        .size(bufferReloadSize)
+                        .trackTotalHits(t -> t.enabled(false))
+                        .query(
+                                q ->
+                                        q.bool(
+                                                b ->
+                                                        b.filter(
+                                                                        f ->
+                                                                                f.range(
+                                                                                        r ->
+                                                                                                r.field(
+                                                                                                                "nextFetchDate")
+                                                                                                        .lte(
+                                                                                                                JsonData
+                                                                                                                        .of(
+                                                                                                                                formattedQueryDate))))
+                                                                .filter(
+                                                                        f ->
+                                                                                f.term(
+                                                                                        t ->
+                                                                                                t.field(
+                                                                                                                partitionField)
+                                                                                                        .value(
+                                                                                                                FieldValue
+                                                                                                                        .of(
+                                                                                                                                queueName))))));
 
         // sort within a bucket
         for (String bsf : bucketSortField) {
-            FieldSortBuilder sorter = SortBuilders.fieldSort(bsf).order(SortOrder.ASC);
-            sourceBuilder.sort(sorter);
+            requestBuilder.sort(s -> s.field(fs -> fs.field(bsf).order(SortOrder.Asc)));
         }
 
         // do we have a search after for this one?
         Object[] searchAfterValues = searchAfterCache.getIfPresent(queueName);
         if (searchAfterValues != null) {
-            sourceBuilder.searchAfter(searchAfterValues);
+            for (Object sav : searchAfterValues) {
+                requestBuilder.searchAfter((FieldValue) sav);
+            }
         }
 
-        SearchRequest request = new SearchRequest(indexName);
-
-        request.source(sourceBuilder);
-
-        // https://www.elastic.co/guide/en/opensearch/reference/current/search-request-preference.html
-        // _shards:2,3
-        // specific shard but ideally a local copy of it
+        // shard preference for routing
         if (shardID != -1) {
-            request.preference("_shards:" + shardID + "|_local");
+            requestBuilder.preference("_shards:" + shardID + "|_local");
         }
 
-        // dump query to log
-        LOG.debug("{} OpenSearch query {} - {}", logIdprefix, queueName, request.toString());
+        SearchRequest request = requestBuilder.build();
 
-        client.searchAsync(request, RequestOptions.DEFAULT, hrl);
+        // dump query to log
+        LOG.debug("{} OpenSearch query {} - {}", logIdprefix, queueName, request);
+
+        CompletableFuture.supplyAsync(
+                        () -> {
+                            try {
+                                return client.search(request, JsonData.class);
+                            } catch (IOException e) {
+                                throw new CompletionException(e);
+                            }
+                        },
+                        queryExecutor)
+                .thenAccept(hrl::handleResponse)
+                .exceptionally(
+                        e -> {
+                            Throwable cause = e instanceof CompletionException ? e.getCause() : e;
+                            LOG.error("Exception with OpenSearch query", cause);
+                            return null;
+                        });
     }
 
     /** Overrides the handling of responses for aggregations. */
     @Override
-    public void onResponse(SearchResponse response) {
+    protected void handleResponse(SearchResponse<JsonData> response) {
         // delete all entries from the searchAfterCache when
         // we get the results from the aggregation spouts
         searchAfterCache.invalidateAll();
-        super.onResponse(response);
+        super.handleResponse(response);
     }
 
     /** The aggregation kindly told us where to start from. */
@@ -163,40 +192,52 @@ public class HybridSpout extends AggregationSpout implements EmptyQueueListener
     }
 
     /** Handling of results for a specific queue. */
-    class HostResultListener implements ActionListener<SearchResponse> {
+    class HostResultListener {
 
-        @Override
-        public void onResponse(SearchResponse response) {
+        /**
+         * Handles the search response for a host-specific query, extracting hits and adding them to
+         * the buffer.
+         *
+         * @param response the search response containing document hits
+         */
+        void handleResponse(SearchResponse<JsonData> response) {
 
             int alreadyprocessed = 0;
             int numDocs = 0;
 
-            SearchHit[] hits = response.getHits().getHits();
+            List<Hit<JsonData>> hits = response.hits().hits();
 
             Object[] sortValues = null;
 
             // retrieve the key for these results
             String key = null;
 
-            for (SearchHit hit : hits) {
+            for (Hit<JsonData> hit : hits) {
                 numDocs++;
+
+                Map<String, Object> source = sourceAsMap(hit.source());
+
                 String pfield = partitionField;
-                Map<String, Object> sourceAsMap = hit.getSourceAsMap();
+                Map<String, Object> fieldSource = source;
                 if (pfield.startsWith("metadata.")) {
-                    sourceAsMap = (Map<String, Object>) sourceAsMap.get("metadata");
+                    @SuppressWarnings("unchecked")
+                    Map<String, Object> metadataMap = (Map<String, Object>) source.get("metadata");
+                    fieldSource = metadataMap;
                     pfield = pfield.substring(9);
                 }
-                Object key_as_object = sourceAsMap.get(pfield);
+                Object key_as_object = fieldSource.get(pfield);
                 if (key_as_object instanceof List) {
-                    if (((List<String>) (key_as_object)).size() == 1) {
-                        key = ((List<String>) key_as_object).get(0);
+                    @SuppressWarnings("unchecked")
+                    List<String> keyList = (List<String>) key_as_object;
+                    if (keyList.size() == 1) {
+                        key = keyList.get(0);
                     }
                 } else {
                     key = key_as_object.toString();
                 }
 
-                sortValues = hit.getSortValues();
-                if (!addHitToBuffer(hit)) {
+                sortValues = hit.sort().toArray();
+                if (!addHitToBuffer(source)) {
                     alreadyprocessed++;
                 }
             }
@@ -214,14 +255,9 @@ public class HybridSpout extends AggregationSpout implements EmptyQueueListener
                     "{} OpenSearch term query returned {} hits  in {} msec with {} already being processed for {}",
                     logIdprefix,
                     numDocs,
-                    response.getTook().getMillis(),
+                    response.took(),
                     alreadyprocessed,
                     key);
         }
-
-        @Override
-        public void onFailure(Exception e) {
-            LOG.error("Exception with OpenSearch query", e);
-        }
     }
 }
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java
index 3e144872..6af7d909 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java
@@ -29,17 +29,15 @@ import org.apache.storm.task.TopologyContext;
 import org.apache.storm.topology.OutputFieldsDeclarer;
 import org.apache.storm.topology.base.BaseRichBolt;
 import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.opensearch.AsyncBulkProcessor;
 import org.apache.stormcrawler.opensearch.Constants;
 import org.apache.stormcrawler.opensearch.IndexCreation;
 import org.apache.stormcrawler.opensearch.OpenSearchConnection;
 import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.action.DocWriteRequest;
-import org.opensearch.action.bulk.BulkItemResponse;
-import org.opensearch.action.bulk.BulkProcessor;
-import org.opensearch.action.bulk.BulkRequest;
-import org.opensearch.action.bulk.BulkResponse;
-import org.opensearch.action.index.IndexRequest;
-import org.opensearch.core.rest.RestStatus;
+import org.opensearch.client.opensearch.core.BulkRequest;
+import org.opensearch.client.opensearch.core.BulkResponse;
+import org.opensearch.client.opensearch.core.bulk.BulkOperation;
+import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -47,7 +45,7 @@ import org.slf4j.LoggerFactory;
  * Sends information about the queues into an OpenSearch index. This has to be connected to a status
  * updater bolt.
  */
-public class QueueBolt extends BaseRichBolt implements BulkProcessor.Listener {
+public class QueueBolt extends BaseRichBolt implements AsyncBulkProcessor.Listener {
 
     private static final Logger LOG = LoggerFactory.getLogger(QueueBolt.class);
 
@@ -140,36 +138,39 @@ public class QueueBolt extends BaseRichBolt implements BulkProcessor.Listener {
         fields.put("key", key);
         fields.put("lastUpdated", Instant.now().toString());
 
-        final IndexRequest indexRequest =
-                new IndexRequest(indexName).source(fields).id(docID).create(true);
+        final BulkOperation op =
+                BulkOperation.of(b -> b.create(c -> c.index(indexName).id(docID).document(fields)));
 
-        connection.addToProcessor(indexRequest);
+        connection.addToProcessor(op);
     }
 
     @Override
     public void beforeBulk(long executionId, BulkRequest request) {
-        LOG.debug("beforeBulk {} with {} actions", executionId, request.numberOfActions());
+        LOG.debug("beforeBulk {} with {} actions", executionId, request.operations().size());
     }
 
     @Override
     public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
-        LOG.debug("afterBulk [{}] with {} responses", executionId, request.numberOfActions());
-        for (BulkItemResponse item : response.getItems()) {
-            final String key = inFlight.remove(item.getId());
+        LOG.debug("afterBulk [{}] with {} responses", executionId, response.items().size());
+        for (BulkResponseItem item : response.items()) {
+            final String key = inFlight.remove(item.id());
             if (key == null) {
                 continue;
             }
-            if (!item.isFailed()) {
+            if (item.error() == null) {
                 knownQueue.put(key, Boolean.TRUE);
-            } else if (RestStatus.CONFLICT.equals(item.getFailure().getStatus())) {
+            } else if (item.status() == 409) {
                 // entry already exists in the index, e.g. written before a worker
                 // restart or evicted from the cache - not an error
                 knownQueue.put(key, Boolean.TRUE);
             } else {
                 // do not cache the key so that the write is retried
                 // when the key is next seen
+                final var error = item.error();
                 LOG.error(
-                        "Failed to index queue entry {}: {}", key, item.getFailure().getMessage());
+                        "Failed to index queue entry {}: {}",
+                        key,
+                        error.reason() != null ? error.reason() : error.type());
             }
         }
     }
@@ -177,9 +178,9 @@ public class QueueBolt extends BaseRichBolt implements BulkProcessor.Listener {
     @Override
     public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
         LOG.error("Exception with bulk {} - failing the whole lot ", executionId, failure);
-        for (DocWriteRequest<?> docWriteRequest : request.requests()) {
+        for (BulkOperation op : request.operations()) {
             // not cached - the writes are retried when the keys are next seen
-            inFlight.remove(docWriteRequest.id());
+            inFlight.remove(OpenSearchConnection.getBulkOperationId(op));
         }
     }
 
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
index f025f582..2dcd76fe 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
@@ -18,7 +18,9 @@
 package org.apache.stormcrawler.opensearch.persistence;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
@@ -31,6 +33,7 @@ import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.metrics.CrawlerMetrics;
 import org.apache.stormcrawler.metrics.ScopedCounter;
 import org.apache.stormcrawler.metrics.ScopedReducedMetric;
+import org.apache.stormcrawler.opensearch.AsyncBulkProcessor;
 import org.apache.stormcrawler.opensearch.Constants;
 import org.apache.stormcrawler.opensearch.IndexCreation;
 import org.apache.stormcrawler.opensearch.OpenSearchConnection;
@@ -39,12 +42,9 @@ import org.apache.stormcrawler.persistence.AbstractStatusUpdaterBolt;
 import org.apache.stormcrawler.persistence.Status;
 import org.apache.stormcrawler.util.ConfUtils;
 import org.apache.stormcrawler.util.URLPartitioner;
-import org.opensearch.action.bulk.BulkProcessor;
-import org.opensearch.action.bulk.BulkRequest;
-import org.opensearch.action.bulk.BulkResponse;
-import org.opensearch.action.index.IndexRequest;
-import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.client.opensearch.core.BulkRequest;
+import org.opensearch.client.opensearch.core.BulkResponse;
+import org.opensearch.client.opensearch.core.bulk.BulkOperation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -52,7 +52,8 @@ import org.slf4j.LoggerFactory;
  * Simple bolt which stores the status of URLs into OpenSearch. Takes the tuples coming from the
  * 'status' stream. To be used in combination with a Spout to read from the index.
  */
-public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt implements BulkProcessor.Listener {
+public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt
+        implements AsyncBulkProcessor.Listener {
 
     private static final Logger LOG = LoggerFactory.getLogger(StatusUpdaterBolt.class);
 
@@ -142,6 +143,16 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt implements Bulk
 
         int metrics_time_bucket_secs = 30;
 
+        // benchmarking - average number of items received back from OpenSearch per second
+        this.receivedPerSecMetrics =
+                CrawlerMetrics.registerPerSecMetric(
+                        context, stormConf, "average_persec", metrics_time_bucket_secs);
+
+        // eventCounter MUST be registered before WaitAckCache — the eviction lambda captures it
+        this.eventCounter =
+                CrawlerMetrics.registerCounter(
+                        context, stormConf, "counters", metrics_time_bucket_secs);
+
         String defaultSpec =
                 String.format(
                         Locale.ROOT,
@@ -159,24 +170,13 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt implements Bulk
                             eventCounter.scope("purged").incrBy(1);
                             collector.fail(t);
                         });
-
-        // create gauge for waitAck
         CrawlerMetrics.registerGauge(
                 context, stormConf, "waitAck", waitAck::estimatedSize, metrics_time_bucket_secs);
 
-        // benchmarking - average number of items received back by Elastic per second
-        this.receivedPerSecMetrics =
-                CrawlerMetrics.registerPerSecMetric(
-                        context, stormConf, "average_persec", metrics_time_bucket_secs);
-
-        this.eventCounter =
-                CrawlerMetrics.registerCounter(
-                        context, stormConf, "counters", metrics_time_bucket_secs);
-
         try {
             connection = OpenSearchConnection.getConnection(stormConf, OSBoltType, this);
         } catch (Exception e1) {
-            LOG.error("Can't connect to ElasticSearch", e1);
+            LOG.error("Can't connect to OpenSearch", e1);
             throw new RuntimeException(e1);
         }
 
@@ -205,8 +205,10 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt implements Bulk
 
         String documentID = getDocumentID(metadata, url);
 
-        // check that the same URL is not being sent to OpenSearch
-        if (status.equals(Status.DISCOVERED) && waitAck.contains(documentID)) {
+        boolean isAlreadySentAndDiscovered =
+                status.equals(Status.DISCOVERED) && waitAck.contains(documentID);
+
+        if (isAlreadySentAndDiscovered) {
             // if this object is discovered - adding another version of it
             // won't make any difference
             LOG.debug(
@@ -220,16 +222,16 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt implements Bulk
             return;
         }
 
-        XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
-        builder.field("url", url);
-        builder.field("status", status);
+        Map<String, Object> doc = new HashMap<>();
+        doc.put("url", url);
+        doc.put("status", status.name());
 
-        builder.startObject("metadata");
+        Map<String, Object> metadataMap = new HashMap<>();
         for (String mdKey : metadata.keySet()) {
             String[] values = metadata.getValues(mdKey);
             // periods are not allowed - replace with %2E
             mdKey = mdKey.replaceAll("\\.", "%2E");
-            builder.array(mdKey, values);
+            metadataMap.put(mdKey, Arrays.asList(values));
         }
 
         String partitionKey = partitioner.getPartition(url, metadata);
@@ -245,62 +247,81 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt implements Bulk
 
         // store routing key in metadata?
         if (StringUtils.isNotBlank(fieldNameForRoutingKey) && routingFieldNameInMetadata) {
-            builder.field(fieldNameForRoutingKey, partitionKey);
+            metadataMap.put(fieldNameForRoutingKey, partitionKey);
         }
 
-        builder.endObject();
+        doc.put("metadata", metadataMap);
 
         // store routing key outside metadata?
         if (StringUtils.isNotBlank(fieldNameForRoutingKey) && !routingFieldNameInMetadata) {
-            builder.field(fieldNameForRoutingKey, partitionKey);
+            doc.put(fieldNameForRoutingKey, partitionKey);
         }
 
         if (nextFetch.isPresent()) {
-            builder.timeField("nextFetchDate", nextFetch.get());
+            doc.put("nextFetchDate", nextFetch.get().toInstant().toString());
         }
-
-        builder.endObject();
-
-        IndexRequest request = new IndexRequest(getIndexName(metadata));
-
         // check that we don't overwrite an existing entry
         // When create is used, the index operation will fail if a document
         // by that id already exists in the index.
         final boolean create = status.equals(Status.DISCOVERED);
-        request.source(builder).id(documentID).create(create);
-
-        if (doRouting) {
-            request.routing(partitionKey);
+        final String targetIndex = getIndexName(metadata);
+        final String routing = doRouting ? partitionKey : null;
+
+        BulkOperation op;
+        if (create) {
+            op =
+                    BulkOperation.of(
+                            b ->
+                                    b.create(
+                                            c -> {
+                                                c.index(targetIndex).id(documentID).document(doc);
+                                                if (routing != null) {
+                                                    c.routing(routing);
+                                                }
+                                                return c;
+                                            }));
+        } else {
+            op =
+                    BulkOperation.of(
+                            b ->
+                                    b.index(
+                                            idx -> {
+                                                idx.index(targetIndex).id(documentID).document(doc);
+                                                if (routing != null) {
+                                                    idx.routing(routing);
+                                                }
+                                                return idx;
+                                            }));
         }
 
         waitAck.addTuple(documentID, tuple);
 
         LOG.debug("Sending to OpenSearch buffer {} with ID {}", url, documentID);
 
-        connection.addToProcessor(request);
+        connection.addToProcessor(op);
     }
 
     @Override
     public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
-        LOG.debug("afterBulk [{}] with {} responses", executionId, request.numberOfActions());
+        LOG.debug("afterBulk [{}] with {} responses", executionId, request.operations().size());
         eventCounter.scope("bulks_received").incrBy(1);
-        eventCounter.scope("bulk_msec").incrBy(response.getTook().getMillis());
-        eventCounter.scope("received").incrBy(request.numberOfActions());
-        receivedPerSecMetrics.scope("received").update(request.numberOfActions());
+        eventCounter.scope("bulk_msec").incrBy(response.took());
+        eventCounter.scope("received").incrBy(request.operations().size());
+        receivedPerSecMetrics.scope("received").update(request.operations().size());
 
         waitAck.processBulkResponse(
                 response,
                 executionId,
                 eventCounter,
-                (id, t, selected) -> {
-                    if (!selected.failed) {
-                        String url = t.getStringByField("url");
+                (id, tuple, selected) -> {
+                    if (!selected.failed()) {
+                        String url = tuple.getStringByField("url");
                         LOG.debug("Acked {} with ID {}", url, id);
                         eventCounter.scope("acked").incrBy(1);
-                        ack(t, url);
+                        super.ack(tuple, url);
                     } else {
                         eventCounter.scope("failed").incrBy(1);
-                        collector.fail(t);
+                        collector.fail(tuple);
                     }
                 });
     }
@@ -308,8 +329,8 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt implements Bulk
     @Override
     public void afterBulk(long executionId, BulkRequest request, Throwable throwable) {
         eventCounter.scope("bulks_received").incrBy(1);
-        eventCounter.scope("received").incrBy(request.numberOfActions());
-        receivedPerSecMetrics.scope("received").update(request.numberOfActions());
+        eventCounter.scope("received").incrBy(request.operations().size());
+        receivedPerSecMetrics.scope("received").update(request.operations().size());
 
         waitAck.processFailedBulk(
                 request,
@@ -323,7 +344,7 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt implements Bulk
 
     @Override
     public void beforeBulk(long executionId, BulkRequest request) {
-        LOG.debug("beforeBulk {} with {} actions", executionId, request.numberOfActions());
+        LOG.debug("beforeBulk {} with {} actions", executionId, request.operations().size());
         eventCounter.scope("bulks_sent").incrBy(1);
     }
 
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorHttpTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorHttpTest.java
similarity index 100%
rename from external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorHttpTest.java
rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorHttpTest.java
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorRetryTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorRetryTest.java
similarity index 100%
rename from external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorRetryTest.java
rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorRetryTest.java
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorTest.java
similarity index 100%
rename from external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorTest.java
rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorTest.java
diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java
index ae90bebd..bb484e84 100644
--- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java
@@ -23,186 +23,95 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.UncheckedIOException;
-import java.net.InetAddress;
-import java.net.ServerSocket;
-import java.net.Socket;
-import java.nio.charset.StandardCharsets;
 import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
-import org.apache.http.HttpHost;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.Credentials;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
+import org.apache.hc.client5.http.auth.AuthScope;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
+import org.apache.hc.core5.http.HttpHost;
 import org.junit.jupiter.api.Test;
-import org.opensearch.client.Request;
-import org.opensearch.client.RestClient;
-import org.opensearch.client.sniff.OpenSearchNodesSniffer;
 
 class OpenSearchConnectionCredentialsTest {
 
-    private static final Credentials CREDENTIALS =
-            new UsernamePasswordCredentials("crawler", "s3cret");
-
-    private static CredentialsProvider providerFor(HttpHost... hosts) {
-        return new OpenSearchConnection.OriginCredentialsProvider(List.of(hosts), CREDENTIALS);
-    }
-
-    /** The scope HttpClient looks the credentials up with for a request to the given node. */
-    private static AuthScope request(String scheme, String host, int port) {
-        return new AuthScope(new HttpHost(host, port, scheme), AuthScope.ANY_REALM, "Basic");
+    private static BasicCredentialsProvider providerFor(List<HttpHost> hosts) {
+        final BasicCredentialsProvider provider = new BasicCredentialsProvider();
+        final UsernamePasswordCredentials credentials =
+                new UsernamePasswordCredentials("crawler", "s3cret".toCharArray());
+        for (AuthScope scope : OpenSearchConnection.credentialScopes(hosts)) {
+            provider.setCredentials(scope, credentials);
+        }
+        return provider;
     }
 
     @Test
-    void credentialsAreGivenToTheConfiguredAddresses() {
-        final CredentialsProvider provider =
-                providerFor(
-                        new HttpHost("opensearch1.example.org", 9200, "https"),
-                        new HttpHost("OpenSearch2.example.org", 9201, "https"));
-        assertEquals(
-                CREDENTIALS,
-                provider.getCredentials(request("https", "opensearch1.example.org", 9200)));
+    void scopesCoverTheConfiguredHostsAndPorts() {
+        final List<AuthScope> scopes =
+                OpenSearchConnection.credentialScopes(
+                        List.of(
+                                new HttpHost("https", "opensearch1.example.org", 9200),
+                                new HttpHost("https", "opensearch2.example.org", 9201)));
         assertEquals(
-                CREDENTIALS,
-                provider.getCredentials(request("HTTPS", "opensearch2.example.org", 9201)));
+                List.of(
+                        new AuthScope("https", "opensearch1.example.org", 9200, null, null),
+                        new AuthScope("https", "opensearch2.example.org", 9201, null, null)),
+                scopes);
     }
 
     @Test
-    void credentialsAreOnlyGivenToTheConfiguredHosts() {
-        final CredentialsProvider provider =
-                providerFor(new HttpHost("opensearch1.example.org", 9200, "https"));
-
-        // a node found by the sniffer under the address it publishes
-        assertNull(provider.getCredentials(request("https", "10.0.0.12", 9200)));
-        // same host, another port
-        assertNull(provider.getCredentials(request("https", "opensearch1.example.org", 9300)));
-        assertNull(provider.getCredentials(request("https", "other.example.org", 9200)));
-    }
-
-    @Test
-    void credentialsAreMatchedOnTheScheme() {
-        final CredentialsProvider provider =
-                providerFor(new HttpHost("opensearch1.example.org", 9200, "https"));
-        assertNull(provider.getCredentials(request("http", "opensearch1.example.org", 9200)));
+    void duplicateAddressesGiveOneScope() {
+        final List<AuthScope> scopes =
+                OpenSearchConnection.credentialScopes(
+                        List.of(
+                                new HttpHost("https", "opensearch1.example.org", 9200),
+                                new HttpHost("https", "OpenSearch1.example.org", 9200)));
+        assertEquals(1, scopes.size());
     }
 
     @Test
-    void credentialsNeedAnOrigin() {
-        final CredentialsProvider provider =
-                providerFor(new HttpHost("opensearch1.example.org", 9200, "https"));
-        assertNull(provider.getCredentials(new AuthScope("opensearch1.example.org", 9200)));
-        assertNull(provider.getCredentials(AuthScope.ANY));
+    void noAddressGivesNoScope() {
+        assertTrue(OpenSearchConnection.credentialScopes(List.of()).isEmpty());
     }
 
     @Test
-    void noAddressGivesTheCredentialsToNoOne() {
-        assertNull(providerFor().getCredentials(request("https", "opensearch1.example.org", 9200)));
-    }
+    void credentialsAreOnlyGivenToTheConfiguredHosts() {
+        final BasicCredentialsProvider provider =
+                providerFor(List.of(new HttpHost("https", "opensearch1.example.org", 9200)));
 
-    /**
-     * Sends a request with the client to a local server and returns the Authorization header it
-     * received, if any.
-     */
-    private static String authorizationSentTo(HttpHost configured) throws Exception {
-        try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getLoopbackAddress())) {
-            final int port = server.getLocalPort();
-            final CompletableFuture<String> authorization =
-                    CompletableFuture.supplyAsync(
-                            () -> {
-                                try (Socket socket = server.accept()) {
-                                    final BufferedReader in =
-                                            new BufferedReader(
-                                                    new InputStreamReader(
-                                                            socket.getInputStream(),
-                                                            StandardCharsets.US_ASCII));
-                                    String header = null;
-                                    for (String line = in.readLine();
-                                            line != null && !line.isEmpty();
-                                            line = in.readLine()) {
-                                        if (line.regionMatches(true, 0, "Authorization:", 0, 14)) {
-                                            header = line.substring(14).trim();
-                                        }
-                                    }
-                                    socket.getOutputStream()
-                                            .write(
-                                                    "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
-                                                            .getBytes(StandardCharsets.US_ASCII));
-                                    return header;
-                                } catch (IOException e) {
-                                    throw new UncheckedIOException(e);
-                                }
-                            });
-            final HttpHost configuredHere =
-                    new HttpHost(configured.getHostName(), port, configured.getSchemeName());
-            try (RestClient client =
-                    RestClient.builder(new HttpHost("127.0.0.1", port, "http"))
-                            .setHttpClientConfigCallback(
-                                    b ->
-                                            b.setDefaultCredentialsProvider(
-                                                    providerFor(configuredHere)))
-                            .build()) {
-                client.performRequest(new Request("GET", "/"));
-            }
-            return authorization.get(10, TimeUnit.SECONDS);
-        }
+        assertNotNull(
+                provider.getCredentials(request("https", "opensearch1.example.org", 9200), null));
+        // another node of the cluster which is not listed in the addresses
+        assertNull(provider.getCredentials(request("https", "10.0.0.12", 9200), null));
+        // same host, another port
+        assertNull(
+                provider.getCredentials(request("https", "opensearch1.example.org", 9300), null));
+        assertNull(provider.getCredentials(request("https", "other.example.org", 9200), null));
     }
 
     @Test
-    void clientSendsTheCredentialsToAConfiguredAddress() throws Exception {
-        assertNotNull(authorizationSentTo(new HttpHost("127.0.0.1", 0, "http")));
+    void credentialsAreMatchedOnTheScheme() {
+        final BasicCredentialsProvider provider =
+                providerFor(List.of(new HttpHost("https", "opensearch1.example.org", 9200)));
+        assertNull(provider.getCredentials(request("http", "opensearch1.example.org", 9200), null));
     }
 
-    @Test
-    void clientSendsNoCredentialsToAnotherAddress() throws Exception {
-        assertNull(authorizationSentTo(new HttpHost("127.0.0.1", 0, "https")));
+    /** The scope HttpClient looks the credentials up with for a request to the given node. */
+    private static AuthScope request(String scheme, String host, int port) {
+        return new AuthScope(new HttpHost(scheme, host, port), null, "Basic");
     }
 
     @Test
     void plainHttpToARemoteHostIsReported() {
-        final HttpHost remote = new HttpHost("opensearch1.example.org", 9200, "http");
+        final HttpHost remote = new HttpHost("http", "opensearch1.example.org", 9200);
         final List<HttpHost> plain =
                 OpenSearchConnection.plainHttpHosts(
                         List.of(
                                 remote,
-                                new HttpHost("opensearch2.example.org", 9200, "https"),
-                                new HttpHost("localhost", 9200, "http"),
-                                new HttpHost("127.0.0.1", 9200, "http"),
-                                new HttpHost("[::1]", 9200, "http")));
+                                new HttpHost("https", "opensearch2.example.org", 9200),
+                                new HttpHost("http", "localhost", 9200),
+                                new HttpHost("http", "127.0.0.1", 9200),
+                                new HttpHost("http", "[::1]", 9200)));
         assertEquals(List.of(remote), plain);
     }
 
-    @Test
-    void sniffedNodesKeepTheSchemeOfTheAddresses() {
-        assertEquals(
-                OpenSearchNodesSniffer.Scheme.HTTPS,
-                OpenSearchConnection.sniffScheme(
-                        List.of(new HttpHost("opensearch1.example.org", 9200, "https"))));
-        assertEquals(
-                OpenSearchNodesSniffer.Scheme.HTTPS,
-                OpenSearchConnection.sniffScheme(
-                        List.of(
-                                new HttpHost("opensearch1.example.org", 9200, "http"),
-                                new HttpHost("opensearch2.example.org", 9200, "https"))));
-        assertEquals(
-                OpenSearchNodesSniffer.Scheme.HTTP,
-                OpenSearchConnection.sniffScheme(
-                        List.of(new HttpHost("opensearch1.example.org", 9200, "http"))));
-    }
-
-    @Test
-    void credentialsNeedUserAndPassword() {
-        assertTrue(OpenSearchConnection.hasCredentials("crawler", "s3cret"));
-        assertFalse(OpenSearchConnection.hasCredentials("crawler", null));
-        assertFalse(OpenSearchConnection.hasCredentials("crawler", " "));
-        assertFalse(OpenSearchConnection.hasCredentials(null, "s3cret"));
-        assertFalse(OpenSearchConnection.hasCredentials("", ""));
-    }
-
     @Test
     void loopbackAddresses() {
         assertTrue(OpenSearchConnection.isLoopback("localhost"));
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java
similarity index 100%
rename from external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java
rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java
diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java
index 0fd22d18..3bd2dc6d 100644
--- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java
@@ -17,31 +17,29 @@
 
 package org.apache.stormcrawler.opensearch;
 
+import static org.awaitility.Awaitility.await;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertSame;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-import com.github.benmanes.caffeine.cache.Ticker;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
 import org.apache.storm.tuple.Tuple;
 import org.apache.stormcrawler.metrics.ScopedCounter;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.opensearch.action.DocWriteRequest;
-import org.opensearch.action.bulk.BulkItemResponse;
-import org.opensearch.action.bulk.BulkRequest;
-import org.opensearch.action.bulk.BulkResponse;
-import org.opensearch.action.delete.DeleteRequest;
-import org.opensearch.action.index.IndexResponse;
-import org.opensearch.core.index.shard.ShardId;
-import org.opensearch.core.rest.RestStatus;
+import org.opensearch.client.opensearch._types.ErrorCause;
+import org.opensearch.client.opensearch.core.BulkRequest;
+import org.opensearch.client.opensearch.core.BulkResponse;
+import org.opensearch.client.opensearch.core.bulk.BulkOperation;
+import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
+import org.opensearch.client.opensearch.core.bulk.OperationType;
 import org.slf4j.LoggerFactory;
 
 class WaitAckCacheTest {
@@ -66,23 +64,33 @@ class WaitAckCacheTest {
         return t;
     }
 
-    private static ShardId shardId() {
-        return new ShardId("index", "_na_", 0);
+    private static BulkResponseItem successItem(String docId) {
+        return BulkResponseItem.of(
+                b -> b.id(docId).index("index").status(200).operationType(OperationType.Index));
     }
 
-    private static BulkItemResponse successItem(int itemId, String docId) {
-        IndexResponse indexResponse = new IndexResponse(shardId(), docId, 1, 1, 1, true);
-        return new BulkItemResponse(itemId, DocWriteRequest.OpType.INDEX, indexResponse);
+    private static BulkResponseItem failedItem(String docId, int status) {
+        return BulkResponseItem.of(
+                b ->
+                        b.id(docId)
+                                .index("index")
+                                .status(status)
+                                .operationType(OperationType.Index)
+                                .error(
+                                        ErrorCause.of(
+                                                e -> e.type("test_error").reason("test failure"))));
     }
 
-    private static BulkItemResponse failedItem(int itemId, String docId, RestStatus status) {
-        BulkItemResponse.Failure failure =
-                new BulkItemResponse.Failure("index", docId, new Exception("test failure"), status);
-        return new BulkItemResponse(itemId, DocWriteRequest.OpType.INDEX, failure);
-    }
-
-    private static BulkResponse bulkResponse(BulkItemResponse... items) {
-        return new BulkResponse(items, 10L);
+    private static BulkResponse bulkResponse(BulkResponseItem... items) {
+        boolean hasErrors = false;
+        for (BulkResponseItem item : items) {
+            if (item.error() != null) {
+                hasErrors = true;
+                break;
+            }
+        }
+        final boolean errors = hasErrors;
+        return BulkResponse.of(b -> b.took(10).errors(errors).items(List.of(items)));
     }
 
     @Test
@@ -110,14 +118,14 @@ class WaitAckCacheTest {
         Tuple t = mockTuple("http://example.com");
         cache.addTuple("doc1", t);
 
-        BulkResponse response = bulkResponse(successItem(0, "doc1"));
+        BulkResponse response = bulkResponse(successItem("doc1"));
 
         cache.processBulkResponse(
                 response,
                 1L,
                 null,
                 (id, tuple, selected) -> {
-                    if (!selected.failed) {
+                    if (!selected.failed()) {
                         acked.add(tuple);
                     } else {
                         failed.add(tuple);
@@ -135,15 +143,14 @@ class WaitAckCacheTest {
         Tuple t = mockTuple("http://example.com");
         cache.addTuple("doc1", t);
 
-        BulkResponse response =
-                bulkResponse(failedItem(0, "doc1", RestStatus.INTERNAL_SERVER_ERROR));
+        BulkResponse response = bulkResponse(failedItem("doc1", 500));
 
         cache.processBulkResponse(
                 response,
                 1L,
                 null,
                 (id, tuple, selected) -> {
-                    if (!selected.failed) {
+                    if (!selected.failed()) {
                         acked.add(tuple);
                     } else {
                         failed.add(tuple);
@@ -160,15 +167,18 @@ class WaitAckCacheTest {
         Tuple t = mockTuple("http://example.com");
         cache.addTuple("doc1", t);
 
-        ScopedCounter counter = scopeName -> incrementBy -> {};
-        BulkResponse response = bulkResponse(failedItem(0, "doc1", RestStatus.CONFLICT));
+        ScopedCounter counter = mock(ScopedCounter.class);
+        ScopedCounter.CountHandle handle = mock(ScopedCounter.CountHandle.class);
+        when(counter.scope("doc_conflicts")).thenReturn(handle);
+
+        BulkResponse response = bulkResponse(failedItem("doc1", 409));
 
         cache.processBulkResponse(
                 response,
                 1L,
                 counter,
                 (id, tuple, selected) -> {
-                    if (!selected.failed) {
+                    if (!selected.failed()) {
                         acked.add(tuple);
                     } else {
                         failed.add(tuple);
@@ -177,6 +187,7 @@ class WaitAckCacheTest {
 
         assertEquals(1, acked.size());
         assertEquals(0, failed.size());
+        verify(handle).incrBy(1);
     }
 
     @Test
@@ -186,7 +197,7 @@ class WaitAckCacheTest {
         cache.addTuple("doc1", t1);
         cache.addTuple("doc1", t2);
 
-        BulkResponse response = bulkResponse(successItem(0, "doc1"));
+        BulkResponse response = bulkResponse(successItem("doc1"));
 
         cache.processBulkResponse(response, 1L, null, (id, tuple, selected) -> acked.add(tuple));
 
@@ -201,17 +212,14 @@ class WaitAckCacheTest {
         Tuple t = mockTuple("http://example.com");
         cache.addTuple("doc1", t);
 
-        BulkResponse response =
-                bulkResponse(
-                        failedItem(0, "doc1", RestStatus.INTERNAL_SERVER_ERROR),
-                        successItem(1, "doc1"));
+        BulkResponse response = bulkResponse(failedItem("doc1", 500), successItem("doc1"));
 
         cache.processBulkResponse(
                 response,
                 1L,
                 null,
                 (id, tuple, selected) -> {
-                    if (!selected.failed) {
+                    if (!selected.failed()) {
                         acked.add(tuple);
                     } else {
                         failed.add(tuple);
@@ -229,9 +237,14 @@ class WaitAckCacheTest {
         cache.addTuple("doc1", t1);
         cache.addTuple("doc2", t2);
 
-        BulkRequest request = new BulkRequest();
-        request.add(new DeleteRequest("index", "doc1"));
-        request.add(new DeleteRequest("index", "doc2"));
+        BulkRequest request =
+                BulkRequest.of(
+                        b ->
+                                b.operations(
+                                        BulkOperation.of(
+                                                o -> o.delete(d -> d.index("index").id("doc1"))),
+                                        BulkOperation.of(
+                                                o -> o.delete(d -> d.index("index").id("doc2")))));
 
         cache.processFailedBulk(request, 1L, new Exception("connection lost"), failed::add);
 
@@ -247,8 +260,17 @@ class WaitAckCacheTest {
         Tuple t = mockTuple("http://example.com");
         cache.addTuple("doc1", t);
 
-        BulkRequest request = new BulkRequest();
-        request.add(new DeleteRequest("index", "doc_unknown"));
+        BulkRequest request =
+                BulkRequest.of(
+                        b ->
+                                b.operations(
+                                        BulkOperation.of(
+                                                o ->
+                                                        o.delete(
+                                                                d ->
+                                                                        d.index("index")
+                                                                                .id(
+                                                                                        "doc_unknown")))));
 
         cache.processFailedBulk(request, 1L, new Exception("test"), failed::add);
 
@@ -259,25 +281,29 @@ class WaitAckCacheTest {
 
     @Test
     void eviction_failsTuplesOnExpiry() {
-        AtomicLong fakeTime = new AtomicLong(0);
-        Ticker fakeTicker = fakeTime::get;
         cache =
                 new WaitAckCache(
                         "expireAfterWrite=1s",
                         LoggerFactory.getLogger(WaitAckCacheTest.class),
-                        evicted::add,
-                        fakeTicker);
+                        evicted::add);
         Tuple t = mockTuple("http://example.com");
         cache.addTuple("doc1", t);
 
-        // Advance past the 1s expiry
-        fakeTime.set(TimeUnit.SECONDS.toNanos(2));
-
-        // Access triggers expiration; cleanUp forces listener execution
-        cache.contains("doc1");
-        cache.cleanUp();
+        // Force cache maintenance after expiry by doing a contains() check
+        // which accesses the cache and triggers Caffeine's cleanup
+        await().atMost(5, TimeUnit.SECONDS)
+                .pollInterval(200, TimeUnit.MILLISECONDS)
+                .untilAsserted(
+                        () -> {
+                            // contains() accesses the cache which triggers cleanup
+                            cache.contains("doc1");
+                            // also try adding and invalidating a dummy entry to force maintenance
+                            Tuple dummy = mockTuple("http://dummy");
+                            cache.addTuple("_probe_", dummy);
+                            cache.invalidate("_probe_");
+                            assertFalse(evicted.isEmpty(), "Eviction callback should have fired");
+                        });
 
-        assertFalse(evicted.isEmpty(), "Eviction callback should have fired");
         assertTrue(evicted.contains(t));
     }
 
@@ -288,17 +314,14 @@ class WaitAckCacheTest {
         cache.addTuple("doc1", t1);
         cache.addTuple("doc2", t2);
 
-        BulkResponse response =
-                bulkResponse(
-                        successItem(0, "doc1"),
-                        failedItem(1, "doc2", RestStatus.INTERNAL_SERVER_ERROR));
+        BulkResponse response = bulkResponse(successItem("doc1"), failedItem("doc2", 500));
 
         cache.processBulkResponse(
                 response,
                 1L,
                 null,
                 (id, tuple, selected) -> {
-                    if (!selected.failed) {
+                    if (!selected.failed()) {
                         acked.add(tuple);
                     } else {
                         failed.add(tuple);
@@ -310,20 +333,4 @@ class WaitAckCacheTest {
         assertEquals(1, failed.size());
         assertSame(t2, failed.get(0));
     }
-
-    @Test
-    void shutdown_failsAllRemainingTuples() {
-        Tuple t1 = mockTuple("http://example.com/1");
-        Tuple t2 = mockTuple("http://example.com/2");
-        cache.addTuple("doc1", t1);
-        cache.addTuple("doc2", t2);
-
-        cache.shutdown();
-
-        assertEquals(2, evicted.size());
-        assertTrue(evicted.contains(t1));
-        assertTrue(evicted.contains(t2));
-        assertFalse(cache.contains("doc1"));
-        assertFalse(cache.contains("doc2"));
-    }
 }
diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
index e9c72b33..44150ea6 100644
--- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
@@ -26,7 +26,8 @@ import org.testcontainers.utility.DockerImageName;
 @Testcontainers(disabledWithoutDocker = true)
 public abstract class AbstractOpenSearchTest {
 
-    private static final String OPENSEARCH_VERSION = "2.19.4";
+    private static final String OPENSEARCH_VERSION =
+            System.getProperty("opensearch-version", "2.19.5");
 
     public static final String PASSWORD = "This1sAPassw0rd";
 
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java
similarity index 100%
rename from external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java
rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java
diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java
index 92b665c5..296c58dd 100644
--- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java
@@ -31,7 +31,7 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import org.apache.http.HttpHost;
+import org.apache.hc.core5.http.HttpHost;
 import org.apache.storm.task.OutputCollector;
 import org.apache.storm.tuple.Tuple;
 import org.apache.stormcrawler.Metadata;
@@ -44,12 +44,11 @@ import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Timeout;
-import org.opensearch.action.get.GetRequest;
-import org.opensearch.action.get.GetResponse;
-import org.opensearch.client.RequestOptions;
-import org.opensearch.client.RestClient;
-import org.opensearch.client.RestClientBuilder;
-import org.opensearch.client.RestHighLevelClient;
+import org.opensearch.client.json.jackson.JacksonJsonpMapper;
+import org.opensearch.client.opensearch.OpenSearchClient;
+import org.opensearch.client.opensearch.core.GetResponse;
+import org.opensearch.client.transport.OpenSearchTransport;
+import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -59,7 +58,9 @@ class QueueBoltTest extends AbstractOpenSearchTest {
 
     protected TestOutputCollector output;
 
-    protected org.opensearch.client.RestHighLevelClient client;
+    protected OpenSearchClient client;
+
+    private OpenSearchTransport transport;
 
     private static final Logger LOG = LoggerFactory.getLogger(QueueBoltTest.class);
 
@@ -78,12 +79,15 @@ class QueueBoltTest extends AbstractOpenSearchTest {
 
     @BeforeEach
     void setupQueueBolt() throws IOException {
-        RestClientBuilder builder =
-                RestClient.builder(
-                        new HttpHost(
-                                opensearchContainer.getHost(),
-                                opensearchContainer.getMappedPort(9200)));
-        client = new RestHighLevelClient(builder);
+        transport =
+                ApacheHttpClient5TransportBuilder.builder(
+                                new HttpHost(
+                                        "http",
+                                        opensearchContainer.getHost(),
+                                        opensearchContainer.getMappedPort(9200)))
+                        .setMapper(new JacksonJsonpMapper())
+                        .build();
+        client = new OpenSearchClient(transport);
         output = new TestOutputCollector();
         bolt = prepareQueueBolt();
     }
@@ -105,7 +109,7 @@ class QueueBoltTest extends AbstractOpenSearchTest {
         bolt.cleanup();
         output = null;
         try {
-            client.close();
+            transport.close();
         } catch (IOException e) {
         }
     }
@@ -129,20 +133,19 @@ class QueueBoltTest extends AbstractOpenSearchTest {
                 .until(
                         () -> {
                             try {
-                                GetResponse result =
-                                        client.get(
-                                                new GetRequest("queues", id),
-                                                RequestOptions.DEFAULT);
-                                return result.isExists();
+                                GetResponse<Map> result =
+                                        client.get(g -> g.index("queues").id(id), Map.class);
+                                return result.found();
                             } catch (IOException e) {
                                 return false;
                             }
                         });
     }
 
+    @SuppressWarnings("unchecked")
     private Map<String, Object> getSource(String key) throws IOException {
         String id = org.apache.commons.codec.digest.DigestUtils.sha256Hex(key);
-        return client.get(new GetRequest("queues", id), RequestOptions.DEFAULT).getSourceAsMap();
+        return client.get(g -> g.index("queues").id(id), Map.class).source();
     }
 
     @Test
diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java
index 19df1453..71fe7795 100644
--- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java
@@ -32,7 +32,7 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import org.apache.http.HttpHost;
+import org.apache.hc.core5.http.HttpHost;
 import org.apache.storm.task.OutputCollector;
 import org.apache.storm.tuple.Tuple;
 import org.apache.stormcrawler.Metadata;
@@ -46,12 +46,11 @@ import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Timeout;
-import org.opensearch.action.get.GetRequest;
-import org.opensearch.action.get.GetResponse;
-import org.opensearch.client.RequestOptions;
-import org.opensearch.client.RestClient;
-import org.opensearch.client.RestClientBuilder;
-import org.opensearch.client.RestHighLevelClient;
+import org.opensearch.client.json.jackson.JacksonJsonpMapper;
+import org.opensearch.client.opensearch.OpenSearchClient;
+import org.opensearch.client.opensearch.core.GetResponse;
+import org.opensearch.client.transport.OpenSearchTransport;
+import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -61,7 +60,9 @@ class StatusBoltTest extends AbstractOpenSearchTest {
 
     protected TestOutputCollector output;
 
-    protected org.opensearch.client.RestHighLevelClient client;
+    protected OpenSearchClient client;
+
+    private OpenSearchTransport transport;
 
     private static final Logger LOG = LoggerFactory.getLogger(StatusBoltTest.class);
 
@@ -81,12 +82,15 @@ class StatusBoltTest extends AbstractOpenSearchTest {
     @BeforeEach
     void setupStatusBolt() throws IOException {
         bolt = new StatusUpdaterBolt();
-        RestClientBuilder builder =
-                RestClient.builder(
-                        new HttpHost(
-                                opensearchContainer.getHost(),
-                                opensearchContainer.getMappedPort(9200)));
-        client = new RestHighLevelClient(builder);
+        transport =
+                ApacheHttpClient5TransportBuilder.builder(
+                                new HttpHost(
+                                        "http",
+                                        opensearchContainer.getHost(),
+                                        opensearchContainer.getMappedPort(9200)))
+                        .setMapper(new JacksonJsonpMapper())
+                        .build();
+        client = new OpenSearchClient(transport);
         // configure the status updater bolt
         Map<String, Object> conf = new HashMap<>();
         conf.put("opensearch.status.routing.fieldname", "metadata.key");
@@ -107,7 +111,7 @@ class StatusBoltTest extends AbstractOpenSearchTest {
         bolt.cleanup();
         output = null;
         try {
-            client.close();
+            transport.close();
         } catch (IOException e) {
         }
     }
@@ -129,6 +133,7 @@ class StatusBoltTest extends AbstractOpenSearchTest {
     @Test
     @Timeout(value = 2, unit = TimeUnit.MINUTES)
     // see https://github.com/apache/stormcrawler/issues/885
+    @SuppressWarnings("unchecked")
     void checkListKeyFromOpensearch()
             throws IOException, ExecutionException, InterruptedException, TimeoutException {
         String url = "https://www.url.net/something";
@@ -136,10 +141,10 @@ class StatusBoltTest extends AbstractOpenSearchTest {
         md.addValue("someKey", "someValue");
         store(url, Status.DISCOVERED, md).get(10, TimeUnit.SECONDS);
         assertEquals(1, output.getAckedTuples().size());
-        // check output in Opensearch?
+        // check output in Opensearch
         String id = org.apache.commons.codec.digest.DigestUtils.sha256Hex(url);
-        GetResponse result = client.get(new GetRequest("status", id), RequestOptions.DEFAULT);
-        Map<String, Object> sourceAsMap = result.getSourceAsMap();
+        GetResponse<Map> result = client.get(g -> g.index("status").id(id), Map.class);
+        Map<String, Object> sourceAsMap = result.source();
         final String pfield = "metadata.somekey";
         sourceAsMap = (Map<String, Object>) sourceAsMap.get("metadata");
         final var pfieldNew = pfield.substring(9);
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpoutTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpoutTest.java
similarity index 100%
rename from external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpoutTest.java
rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpoutTest.java


(stormcrawler) 07/07: keep the connection pool limits when TLS validation is disabled

Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

dpol1 pushed a commit to branch opensearch-3.x
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git

commit baef68faa840df5b7e5ed44ad423e01c96e4ed8b
Author: Davide Polato <dp...@apache.org>
AuthorDate: Fri Oct 16 16:44:24 2026 +0200

    keep the connection pool limits when TLS validation is disabled
    
    With opensearch.disable.tls.validation set, OpenSearchConnection
    replaces the connection manager of the transport to install a
    trust-all TLS strategy. The replacement had no pool limits, so
    HttpClient 5 allowed 5 connections per node and 25 in total, instead
    of the 10 and 30 that the transport sets.
    
    The replacement now uses the transport's limits. The new test sends
    12 requests to a local server that never answers and checks that 10
    connections are open.
    
    Signed-off-by: Davide Polato <dp...@apache.org>
---
 .../opensearch/OpenSearchConnection.java           |  6 ++-
 .../opensearch/OpenSearchConnectionTest.java       | 62 ++++++++++++++++++++++
 2 files changed, 67 insertions(+), 1 deletion(-)

diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
index f822c6d7..d8ca3011 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
@@ -18,6 +18,8 @@
 package org.apache.stormcrawler.opensearch;
 
 import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_CONNECT_TIMEOUT_MILLIS;
+import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_MAX_CONN_PER_ROUTE;
+import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_MAX_CONN_TOTAL;
 import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_RESPONSE_TIMEOUT_MILLIS;
 
 import java.io.IOException;
@@ -435,7 +437,7 @@ public final class OpenSearchConnection {
                                     new HttpHost(proxyscheme, proxyhost, proxyport));
                         }
                         // Custom connection manager overrides the builder's internal one,
-                        // so timeouts and TlsDetailsFactory must be replicated here
+                        // so pool limits, timeouts and TlsDetailsFactory must be replicated here
                         if (disableTlsValidation) {
                             try {
                                 final SSLContext sslContext =
@@ -444,6 +446,8 @@ public final class OpenSearchConnection {
                                                 .build();
                                 httpClientBuilder.setConnectionManager(
                                         PoolingAsyncClientConnectionManagerBuilder.create()
+                                                .setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE)
+                                                .setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL)
                                                 .setTlsStrategy(
                                                         ClientTlsStrategyBuilder.create()
                                                                 .setSslContext(sslContext)
diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java
index 8c5789a1..19b39ce6 100644
--- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java
@@ -17,9 +17,23 @@
 
 package org.apache.stormcrawler.opensearch;
 
+import static org.awaitility.Awaitility.await;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import org.junit.jupiter.api.Test;
+import org.opensearch.client.opensearch.OpenSearchClient;
+import org.opensearch.client.opensearch.generic.Requests;
 
 class OpenSearchConnectionTest {
 
@@ -72,4 +86,52 @@ class OpenSearchConnectionTest {
     void whitespaceIsTrimmed() {
         assertEquals(5000, OpenSearchConnection.parseTimeValueToMillis("  5s  ", 0));
     }
+
+    @Test
+    void disabledTlsValidationKeepsTenConnectionsPerNode() throws Exception {
+        try (ServerSocket server = new ServerSocket(0, 50, InetAddress.getLoopbackAddress())) {
+            // accepts the connections and never answers, so each request keeps its connection
+            final List<Socket> accepted = new CopyOnWriteArrayList<>();
+            final Thread acceptor =
+                    new Thread(
+                            () -> {
+                                try {
+                                    while (true) {
+                                        accepted.add(server.accept());
+                                    }
+                                } catch (IOException e) {
+                                    // server closed
+                                }
+                            });
+            acceptor.setDaemon(true);
+            acceptor.start();
+
+            final Map<String, Object> conf = new HashMap<>();
+            conf.put("opensearch.test.addresses", "http://127.0.0.1:" + server.getLocalPort());
+            conf.put("opensearch.disable.tls.validation", true);
+            final OpenSearchClient client = OpenSearchConnection.getClient(conf, "test");
+            final ExecutorService requests = Executors.newFixedThreadPool(12);
+            try {
+                for (int i = 0; i < 12; i++) {
+                    requests.submit(
+                            () ->
+                                    client.generic()
+                                            .execute(
+                                                    Requests.builder()
+                                                            .endpoint("/")
+                                                            .method("GET")
+                                                            .build()));
+                }
+                await().during(Duration.ofMillis(300))
+                        .atMost(Duration.ofSeconds(5))
+                        .untilAsserted(() -> assertEquals(10, accepted.size()));
+            } finally {
+                for (Socket socket : accepted) {
+                    socket.close();
+                }
+                requests.shutdownNow();
+                client._transport().close();
+            }
+        }
+    }
 }


(stormcrawler) 04/07: drop the sniff and responseBufferSize settings

Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

dpol1 pushed a commit to branch opensearch-3.x
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git

commit dade9a7f6bea9c6b82974f40a37a2b558e525785
Author: Davide Polato <dp...@apache.org>
AuthorDate: Fri Oct 16 15:58:25 2026 +0200

    drop the sniff and responseBufferSize settings
    
    The opensearch-java client has no sniffer and no per-request response
    buffer setting. The keys are removed from the module's
    opensearch-conf.yaml, from the one of the archetype and from the
    configuration reference. When they are set, OpenSearchConnection
    still logs a WARN, which now points to external/opensearch/README.md.
    
    The configuration reference and the Javadoc of DelegateRefresher no
    longer mention a second OpenSearch module.
    
    Signed-off-by: Davide Polato <dp...@apache.org>
---
 docs/src/main/asciidoc/configuration.adoc                  |  7 +++----
 .../resources/archetype-resources/opensearch-conf.yaml     |  4 ----
 external/opensearch/opensearch-conf.yaml                   | 13 -------------
 .../apache/stormcrawler/opensearch/DelegateRefresher.java  |  4 ----
 .../stormcrawler/opensearch/OpenSearchConnection.java      | 14 +++++++-------
 5 files changed, 10 insertions(+), 32 deletions(-)

diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc
index 7ec62c89..f628335b 100644
--- a/docs/src/main/asciidoc/configuration.adoc
+++ b/docs/src/main/asciidoc/configuration.adoc
@@ -355,8 +355,8 @@ Note: `indexer.md.docid` is never used raw: the matching value is passed through
 (the same digest used for the default URL-based ID), except for CloudSearch, whose
 `CloudSearchIndexerBolt` re-digests it with SHA-512 via `CloudSearchUtils.getID` to satisfy
 CloudSearch's ID constraints. It is only honoured by backends whose bolts call
-`AbstractIndexerBolt#getDocumentID` under the hood: the OpenSearch and OpenSearch-Java
-`IndexerBolt`/`DeletionBolt` pairs, and the CloudSearch `CloudSearchIndexerBolt`. The Solr and
+`AbstractIndexerBolt#getDocumentID` under the hood: the OpenSearch
+`IndexerBolt`/`DeletionBolt` pair, and the CloudSearch `CloudSearchIndexerBolt`. The Solr and
 SQL indexers build their document/row key directly from the URL and never call
 `getDocumentID()`, so this setting is silently ignored for those two backends.
 
@@ -443,9 +443,8 @@ See the link:https://github.com/apache/stormcrawler/tree/main/external/opensearc
 | key | default value | description
 
 | opensearch.addresses | - | OpenSearch server address(es). An address without a scheme is contacted over plain http, on port 9200 if none is given; use `https://` when credentials are configured.
-| opensearch.user | - | Username for Basic authentication (optional). The credentials are only sent to the addresses listed, matched on scheme, host and port, not to nodes discovered by sniffing under another address; list those nodes in the addresses or disable `opensearch.<type>.sniff`, which is on by default. A warning is logged when sniffing is enabled together with credentials, and if an address other than a loopback one uses plain http.
+| opensearch.user | - | Username for Basic authentication (optional). The credentials are only sent to the addresses listed, matched on scheme, host and port. A warning is logged if an address other than a loopback one uses plain http.
 | opensearch.password | - | Password for authentication (optional).
-| opensearch.<type>.sniff | true | Discover the other nodes of the cluster and send requests to them as well. The nodes found are contacted over https if one of the addresses uses https, over plain http otherwise.
 | opensearch.disable.tls.validation | false | Do not check the certificates and host names of the OpenSearch nodes. A warning is logged when this is enabled.
 | opensearch.concurrentRequests | 2 | Number of concurrent bulk requests.
 | opensearch.indexer.index.name | content | Index name for crawled documents.
diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml b/external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml
index d1b8d450..6152b943 100644
--- a/external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml
+++ b/external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml
@@ -42,17 +42,14 @@ config:
   opensearch.indexer.bulkActions: 100
   opensearch.indexer.flushInterval: "2s"
   opensearch.indexer.concurrentRequests: 1
-  opensearch.indexer.sniff: true
 
   # MetricsConsumer
   # opensearch.metrics.addresses: "http://localhost:9200"
   opensearch.metrics.index.name: "metrics"
-  opensearch.metrics.sniff: true
 
   # Queues index
   opensearch.queues.addresses: "http://localhost:9200"
   opensearch.queues.index.name: "queues"
-  opensearch.queues.sniff: true
   # cache of known queues, see https://github.com/ben-manes/caffeine/wiki/Specification
   # opensearch.queues.cache.spec: "maximumSize=10000"
 
@@ -70,7 +67,6 @@ config:
   opensearch.status.bulkActions: 500
   opensearch.status.flushInterval: "5s"
   opensearch.status.concurrentRequests: 1
-  opensearch.status.sniff: true
 
     # spout config #
 
diff --git a/external/opensearch/opensearch-conf.yaml b/external/opensearch/opensearch-conf.yaml
index 595e170c..4a31a14c 100644
--- a/external/opensearch/opensearch-conf.yaml
+++ b/external/opensearch/opensearch-conf.yaml
@@ -28,9 +28,6 @@ config:
   #opensearch.password: "PASSWORD"
   opensearch.concurrentRequests: 2
 
-  # Sets the response buffer to the specified value in MB.
-  # opensearch.responseBufferSize: 100
-
   # Disable TLS validation for connection to OpenSearch, a warning is logged
   # when this is enabled
   # opensearch.disable.tls.validation: false
@@ -45,22 +42,15 @@ config:
   opensearch.indexer.bulkActions: 100
   opensearch.indexer.flushInterval: "2s"
   opensearch.indexer.concurrentRequests: 1
-  opensearch.indexer.sniff: true
-  # Sets the response buffer to the specified value in MB.
-  # opensearch.indexer.responseBufferSize: 100
 
   # MetricsConsumer
   # opensearch.metrics.addresses: "http://localhost:9200"
   opensearch.metrics.index.name: "metrics"
-  opensearch.metrics.sniff: true
-  # Sets the response buffer to the specified value in MB.
-  # opensearch.metrics.responseBufferSize: 100
 
 
   # Queues index
   opensearch.queues.addresses: "http://localhost:9200"
   opensearch.queues.index.name: "queues"
-  opensearch.queues.sniff: true
   # cache of known queues, see https://github.com/ben-manes/caffeine/wiki/Specification
   # opensearch.queues.cache.spec: "maximumSize=10000"
 
@@ -78,9 +68,6 @@ config:
   opensearch.status.bulkActions: 500
   opensearch.status.flushInterval: "5s"
   opensearch.status.concurrentRequests: 1
-  opensearch.status.sniff: true
-  # Sets the response buffer to the specified value in MB.
-  # opensearch.status.responseBufferSize: 100
 
     # spout config #
 
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java
index fb03a9f0..7c49745b 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java
@@ -38,10 +38,6 @@ import org.slf4j.LoggerFactory;
  * org.apache.stormcrawler.opensearch.parse.filter.JSONResourceWrapper} to eliminate duplicated
  * setup/refresh/cleanup logic.
  *
- * <p>This is the opensearch-java (OpenSearch Java Client 3.x / HC5) counterpart of the class with
- * the same name in the {@code external/opensearch} module. It uses the typed {@link
- * OpenSearchClient} instead of the deprecated {@code RestHighLevelClient}.
- *
  * @param <T> the base type that the delegate must extend (e.g. URLFilter or ParseFilter)
  */
 public class DelegateRefresher<T> {
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
index eeb47362..f822c6d7 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
@@ -276,29 +276,29 @@ public final class OpenSearchConnection {
     private record ClientResources(OpenSearchClient client, OpenSearchTransport transport) {}
 
     /**
-     * Logs a WARN for legacy configuration keys that are no longer honoured by this module, so that
-     * users migrating from {@code external/opensearch} notice silently-dropped tuning. See the
-     * module README for the full list of differences.
+     * Logs a WARN for configuration keys that earlier releases of this module read and this one
+     * ignores, so that users upgrading notice silently-dropped tuning. See the module README for
+     * the full list of differences.
      */
     private static void warnOnRemovedKeys(Map<String, Object> stormConf, String dottedType) {
         final String responseBufferKey = Constants.PARAMPREFIX + dottedType + "responseBufferSize";
         if (stormConf.containsKey(responseBufferKey)) {
             LOG.warn(
-                    "Configuration key '{}' is set but no longer supported by the opensearch-java module. "
+                    "Configuration key '{}' is set but no longer supported. "
                             + "The HC5-based async transport does not expose an equivalent per-request "
                             + "heap-buffer override. The setting is ignored — remove it from your "
-                            + "configuration. See external/opensearch-java/README.md for details.",
+                            + "configuration. See external/opensearch/README.md for details.",
                     responseBufferKey);
         }
 
         final String sniffKey = Constants.PARAMPREFIX + dottedType + "sniff";
         if (stormConf.containsKey(sniffKey)) {
             LOG.warn(
-                    "Configuration key '{}' is set but no longer supported by the opensearch-java module. "
+                    "Configuration key '{}' is set but no longer supported. "
                             + "The OpenSearch Java Client 3.x does not ship a Sniffer equivalent, so "
                             + "automatic node discovery is not available. Keep the 'addresses' list up to "
                             + "date manually or put a load balancer in front of the cluster. "
-                            + "See external/opensearch-java/README.md for details.",
+                            + "See external/opensearch/README.md for details.",
                     sniffKey);
         }
     }


(stormcrawler) 03/07: build external/opensearch with the opensearch-java client

Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

dpol1 pushed a commit to branch opensearch-3.x
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git

commit a1e375e8ca13f9a1a222a62d453586bee4c453fe
Author: Davide Polato <dp...@apache.org>
AuthorDate: Fri Oct 16 15:56:12 2026 +0200

    build external/opensearch with the opensearch-java client
    
    The module depends on opensearch-java 3.10.0 instead of the
    high-level REST client and the sniffer, and its tests run against
    OpenSearch 3.5.0. The artifactId stays stormcrawler-opensearch.
    
    external/opensearch-java leaves the reactor; its README is merged
    into the one of external/opensearch in a later commit.
    
    Signed-off-by: Davide Polato <dp...@apache.org>
---
 external/opensearch-java/pom.xml | 114 ---------------------------------------
 external/opensearch/pom.xml      |  17 ++----
 pom.xml                          |   1 -
 3 files changed, 5 insertions(+), 127 deletions(-)

diff --git a/external/opensearch-java/pom.xml b/external/opensearch-java/pom.xml
deleted file mode 100644
index 3a12b430..00000000
--- a/external/opensearch-java/pom.xml
+++ /dev/null
@@ -1,114 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied.  See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>org.apache.stormcrawler</groupId>
-        <artifactId>stormcrawler-external</artifactId>
-        <version>4.0.0-SNAPSHOT</version>
-        <relativePath>../pom.xml</relativePath>
-    </parent>
-
-    <properties>
-        <opensearch.server.version>3.5.0</opensearch.server.version>
-        <opensearch.java.version>3.10.0</opensearch.java.version>
-        <jacoco.haltOnFailure>true</jacoco.haltOnFailure>
-        <jacoco.classRatio>0.27</jacoco.classRatio>
-        <jacoco.instructionRatio>0.27</jacoco.instructionRatio>
-        <jacoco.methodRatio>0.25</jacoco.methodRatio>
-        <jacoco.branchRatio>0.17</jacoco.branchRatio>
-        <jacoco.lineRatio>0.29</jacoco.lineRatio>
-        <jacoco.complexityRatio>0.13</jacoco.complexityRatio>
-    </properties>
-
-    <artifactId>stormcrawler-opensearch-java</artifactId>
-    <packaging>jar</packaging>
-
-    <name>stormcrawler-opensearch-java</name>
-    <url>
-        https://github.com/apache/stormcrawler/tree/master/external/opensearch-java</url>
-    <description>OpenSearch module for Apache StormCrawler using the new opensearch-java client</description>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>default-test</id>
-                        <phase>test</phase>
-                        <goals>
-                            <goal>test</goal>
-                        </goals>
-                    </execution>
-                </executions>
-                <configuration>
-                    <systemPropertyVariables>
-                        <opensearch-version>${opensearch.server.version}</opensearch-version>
-                    </systemPropertyVariables>
-                </configuration>
-            </plugin>
-        </plugins>
-    </build>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.opensearch.client</groupId>
-            <artifactId>opensearch-java</artifactId>
-            <version>${opensearch.java.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.stormcrawler</groupId>
-            <artifactId>stormcrawler-core</artifactId>
-            <version>${project.version}</version>
-            <type>test-jar</type>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.testcontainers</groupId>
-            <artifactId>testcontainers</artifactId>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.testcontainers</groupId>
-            <artifactId>junit-jupiter</artifactId>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.awaitility</groupId>
-            <artifactId>awaitility</artifactId>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-simple</artifactId>
-            <scope>test</scope>
-        </dependency>
-
-    </dependencies>
-</project>
diff --git a/external/opensearch/pom.xml b/external/opensearch/pom.xml
index 7813de10..8f0765fb 100644
--- a/external/opensearch/pom.xml
+++ b/external/opensearch/pom.xml
@@ -30,7 +30,8 @@ under the License.
     </parent>
 
     <properties>
-        <opensearch.version>2.19.6</opensearch.version>
+        <opensearch.server.version>3.5.0</opensearch.server.version>
+        <opensearch.java.version>3.10.0</opensearch.java.version>
         <jacoco.haltOnFailure>true</jacoco.haltOnFailure>
         <jacoco.classRatio>0.27</jacoco.classRatio>
         <jacoco.instructionRatio>0.27</jacoco.instructionRatio>
@@ -63,7 +64,7 @@ under the License.
                 </executions>
                 <configuration>
                     <systemPropertyVariables>
-                        <opensearch-version>${opensearch.version}</opensearch-version>
+                        <opensearch-version>${opensearch.server.version}</opensearch-version>
                     </systemPropertyVariables>
                 </configuration>
             </plugin>
@@ -73,8 +74,8 @@ under the License.
     <dependencies>
         <dependency>
             <groupId>org.opensearch.client</groupId>
-            <artifactId>opensearch-rest-high-level-client</artifactId>
-            <version>${opensearch.version}</version>
+            <artifactId>opensearch-java</artifactId>
+            <version>${opensearch.java.version}</version>
         </dependency>
 
         <dependency>
@@ -83,14 +84,6 @@ under the License.
             <scope>test</scope>
         </dependency>
 
-        <!--
-        https://mvnrepository.com/artifact/org.opensearch.client/opensearch-rest-client-sniffer -->
-        <dependency>
-            <groupId>org.opensearch.client</groupId>
-            <artifactId>opensearch-rest-client-sniffer</artifactId>
-            <version>${opensearch.version}</version>
-        </dependency>
-
         <dependency>
             <groupId>org.apache.stormcrawler</groupId>
             <artifactId>stormcrawler-core</artifactId>
diff --git a/pom.xml b/pom.xml
index 2e24ae53..6fd67432 100644
--- a/pom.xml
+++ b/pom.xml
@@ -747,7 +747,6 @@ under the License.
         <module>external/aws</module>
         <module>external/langid</module>
         <module>external/opensearch</module>
-        <module>external/opensearch-java</module>
         <module>external/playwright</module>
         <module>external/solr</module>
         <module>external/sql</module>