You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by dp...@apache.org on 2026/10/16 15:37:08 UTC
(stormcrawler) 02/07: replace the legacy opensearch sources with the opensearch-java ones
This is an automated email from the ASF dual-hosted git repository.
dpol1 pushed a commit to branch opensearch-3.x
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
commit 1ba7227ad600ce8778b64315e0a4bc74ff820743
Author: Davide Polato <dp...@apache.org>
AuthorDate: Fri Oct 16 16:13:49 2026 +0200
replace the legacy opensearch sources with the opensearch-java ones
The files of external/opensearch/src that also exist in
external/opensearch-java take the content of their opensearch-java
counterparts in place, so git blame keeps the lines that did not
change. The files that exist only in external/opensearch-java are
moved. The following commits update the module's POM, configuration
and README.
Signed-off-by: Davide Polato <dp...@apache.org>
---
.../opensearch/BulkItemResponseToFailedFlag.java | 71 --
.../apache/stormcrawler/opensearch/Constants.java | 23 -
.../stormcrawler/opensearch/DelegateRefresher.java | 181 ------
.../stormcrawler/opensearch/IndexCreation.java | 116 ----
.../opensearch/OpenSearchConnection.java | 516 ---------------
.../stormcrawler/opensearch/WaitAckCache.java | 340 ----------
.../stormcrawler/opensearch/bolt/DeletionBolt.java | 180 ------
.../stormcrawler/opensearch/bolt/IndexerBolt.java | 297 ---------
.../opensearch/filtering/JSONURLFilterWrapper.java | 83 ---
.../opensearch/metrics/MetricsConsumer.java | 163 -----
.../opensearch/metrics/MetricsReporter.java | 219 -------
.../opensearch/metrics/StatusMetricsBolt.java | 185 ------
.../parse/filter/JSONResourceWrapper.java | 79 ---
.../opensearch/persistence/AbstractSpout.java | 330 ----------
.../opensearch/persistence/AggregationSpout.java | 432 -------------
.../opensearch/persistence/HybridSpout.java | 263 --------
.../opensearch/persistence/QueueBolt.java | 191 ------
.../opensearch/persistence/StatusUpdaterBolt.java | 358 -----------
.../opensearch/DelegateRefresherTest.java | 256 --------
.../OpenSearchConnectionCredentialsTest.java | 128 ----
.../stormcrawler/opensearch/WaitAckCacheTest.java | 336 ----------
.../opensearch/bolt/AbstractOpenSearchTest.java | 53 --
.../opensearch/bolt/IndexerBoltTest.java | 138 ----
.../opensearch/bolt/QueueBoltTest.java | 202 ------
.../opensearch/bolt/StatusBoltTest.java | 176 -----
.../opensearch/metrics/MetricsReporterTest.java | 54 --
.../src/test/resources/indexer.mapping | 40 --
.../src/test/resources/metrics.mapping | 40 --
.../src/test/resources/queues.mapping | 33 -
.../src/test/resources/status.mapping | 39 --
.../opensearch/AsyncBulkProcessor.java | 0
.../opensearch/BulkItemResponseToFailedFlag.java | 139 ++--
.../stormcrawler/opensearch/DelegateRefresher.java | 33 +-
.../stormcrawler/opensearch/IndexCreation.java | 84 +--
.../opensearch/OpenSearchConnection.java | 715 +++++++++++----------
.../stormcrawler/opensearch/WaitAckCache.java | 59 +-
.../stormcrawler/opensearch/bolt/DeletionBolt.java | 22 +-
.../stormcrawler/opensearch/bolt/IndexerBolt.java | 83 ++-
.../opensearch/filtering/JSONURLFilterWrapper.java | 9 +-
.../opensearch/metrics/MetricsConsumer.java | 33 +-
.../opensearch/metrics/MetricsReporter.java | 32 +-
.../opensearch/metrics/StatusMetricsBolt.java | 135 ++--
.../parse/filter/JSONResourceWrapper.java | 9 +-
.../opensearch/persistence/AbstractSpout.java | 107 ++-
.../opensearch/persistence/AggregationSpout.java | 277 ++++----
.../opensearch/persistence/HybridSpout.java | 162 +++--
.../opensearch/persistence/QueueBolt.java | 41 +-
.../opensearch/persistence/StatusUpdaterBolt.java | 125 ++--
.../opensearch/AsyncBulkProcessorHttpTest.java | 0
.../opensearch/AsyncBulkProcessorRetryTest.java | 0
.../opensearch/AsyncBulkProcessorTest.java | 0
.../OpenSearchConnectionCredentialsTest.java | 199 ++----
.../opensearch/OpenSearchConnectionTest.java | 0
.../stormcrawler/opensearch/WaitAckCacheTest.java | 155 ++---
.../opensearch/bolt/AbstractOpenSearchTest.java | 3 +-
.../opensearch/bolt/NullMetadataValueTest.java | 0
.../opensearch/bolt/QueueBoltTest.java | 45 +-
.../opensearch/bolt/StatusBoltTest.java | 41 +-
.../opensearch/persistence/AbstractSpoutTest.java | 0
59 files changed, 1323 insertions(+), 6707 deletions(-)
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
deleted file mode 100644
index c32f162f..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch;
-
-import java.util.Objects;
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-import org.opensearch.client.opensearch._types.ErrorCause;
-import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
-
-/**
- * Wraps a {@link BulkResponseItem} with a pre-computed failure flag. A 409 (conflict) is not
- * considered a failure — it simply indicates a document already existed when using create mode.
- *
- * @param response the original bulk response item
- * @param failed whether this item represents a real failure (excludes 409 conflicts)
- * @param id the document id from the response item
- */
-public record BulkItemResponseToFailedFlag(
- @NotNull BulkResponseItem response, boolean failed, @NotNull String id) {
-
- public BulkItemResponseToFailedFlag {
- Objects.requireNonNull(response, "response");
- Objects.requireNonNull(id, "id");
- }
-
- /** Constructs with id derived from the response item. */
- public BulkItemResponseToFailedFlag(@NotNull BulkResponseItem response, boolean failed) {
- this(
- response,
- failed,
- Objects.requireNonNull(response.id(), "BulkResponseItem id must not be null"));
- }
-
- /** Returns the error cause, or {@code null} if the item did not fail. */
- @Nullable
- public ErrorCause getFailedCause() {
- return response.error();
- }
-
- /** Returns a human-readable failure description, or {@code null} if the item did not fail. */
- @Nullable
- public String getFailure() {
- ErrorCause error = response.error();
- if (error == null) {
- return null;
- }
- return error.reason() != null ? error.reason() : error.type();
- }
-
- // opensearch-java: status() returns int HTTP code, not RestStatus enum
- /** Returns the HTTP status code of this response item. */
- public int getStatus() {
- return response.status();
- }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/Constants.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/Constants.java
deleted file mode 100644
index 8c0cbc98..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/Constants.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch;
-
-public interface Constants {
-
- String PARAMPREFIX = "opensearch.";
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java
deleted file mode 100644
index fb03a9f0..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Map;
-import java.util.Timer;
-import java.util.TimerTask;
-import org.apache.stormcrawler.JSONResource;
-import org.opensearch.client.json.JsonData;
-import org.opensearch.client.opensearch.OpenSearchClient;
-import org.opensearch.client.opensearch.core.GetResponse;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Loads a delegate class that implements both a required base type and {@link JSONResource}, then
- * periodically refreshes its configuration from OpenSearch. Used by {@link
- * org.apache.stormcrawler.opensearch.filtering.JSONURLFilterWrapper} and {@link
- * org.apache.stormcrawler.opensearch.parse.filter.JSONResourceWrapper} to eliminate duplicated
- * setup/refresh/cleanup logic.
- *
- * <p>This is the opensearch-java (OpenSearch Java Client 3.x / HC5) counterpart of the class with
- * the same name in the {@code external/opensearch} module. It uses the typed {@link
- * OpenSearchClient} instead of the deprecated {@code RestHighLevelClient}.
- *
- * @param <T> the base type that the delegate must extend (e.g. URLFilter or ParseFilter)
- */
-public class DelegateRefresher<T> {
-
- private static final Logger LOG = LoggerFactory.getLogger(DelegateRefresher.class);
-
- private final T delegate;
- private Timer refreshTimer;
- private OpenSearchClient osClient;
-
- /**
- * Creates a refresher by loading the delegate class from the JSON configuration.
- *
- * @param baseType the required base class (e.g. URLFilter.class or ParseFilter.class)
- * @param stormConf the Storm configuration map
- * @param filterParams the JSON params node containing "delegate" and optional "refresh"
- * @param configurer callback to configure the delegate after instantiation
- */
- public DelegateRefresher(
- Class<T> baseType,
- Map<String, Object> stormConf,
- JsonNode filterParams,
- DelegateConfigure<T> configurer) {
-
- JsonNode delegateNode = filterParams.get("delegate");
- if (delegateNode == null) {
- throw new RuntimeException("delegateNode undefined!");
- }
-
- String delegateClassName = null;
- JsonNode node = delegateNode.get("class");
- if (node != null && node.isTextual()) {
- delegateClassName = node.asText();
- }
- if (delegateClassName == null) {
- throw new RuntimeException(baseType.getSimpleName() + " delegate class undefined!");
- }
-
- try {
- Class<?> filterClass = Class.forName(delegateClassName);
-
- if (!baseType.isAssignableFrom(filterClass)) {
- throw new RuntimeException(
- "Filter " + delegateClassName + " does not extend " + baseType.getName());
- }
-
- @SuppressWarnings("unchecked")
- T instance = (T) filterClass.getDeclaredConstructor().newInstance();
-
- if (!(instance instanceof JSONResource)) {
- throw new RuntimeException(
- "Filter " + delegateClassName + " does not implement JSONResource");
- }
-
- this.delegate = instance;
- } catch (RuntimeException e) {
- throw e;
- } catch (Exception e) {
- LOG.error("Can't setup {}: {}", delegateClassName, e);
- throw new RuntimeException("Can't setup " + delegateClassName, e);
- }
-
- // configure the delegate
- JsonNode paramsNode = delegateNode.get("params");
- configurer.configure(delegate, stormConf, paramsNode);
-
- // set up periodic refresh from OpenSearch
- int refreshRate = 600;
- node = filterParams.get("refresh");
- if (node != null && (node.isInt() || node.isTextual())) {
- refreshRate = node.asInt(refreshRate);
- }
-
- final JSONResource resource = (JSONResource) delegate;
-
- refreshTimer = new Timer();
- refreshTimer.schedule(
- new TimerTask() {
- public void run() {
- if (osClient == null) {
- try {
- osClient = OpenSearchConnection.getClient(stormConf, "config");
- } catch (Exception e) {
- LOG.error("Exception while creating OpenSearch connection", e);
- }
- }
- if (osClient != null) {
- LOG.info("Reloading json resources from OpenSearch");
- try {
- GetResponse<JsonData> response =
- osClient.get(
- g ->
- g.index("config")
- .id(resource.getResourceFile()),
- JsonData.class);
- if (response.found() && response.source() != null) {
- String json = response.source().toJson().toString();
- resource.loadJSONResources(
- new ByteArrayInputStream(
- json.getBytes(StandardCharsets.UTF_8)));
- }
- } catch (Exception e) {
- LOG.error("Can't load config from OpenSearch", e);
- }
- }
- }
- },
- refreshRate * 1000L,
- refreshRate * 1000L);
- }
-
- /** Returns the delegate instance. */
- public T getDelegate() {
- return delegate;
- }
-
- /** Cancels the refresh timer and closes the OpenSearch client. */
- public void cleanup() {
- if (refreshTimer != null) {
- refreshTimer.cancel();
- }
- if (osClient != null) {
- try {
- osClient._transport().close();
- } catch (IOException e) {
- LOG.error("Exception when closing OpenSearch client", e);
- }
- osClient = null;
- }
- }
-
- /** Callback interface for configuring the delegate after instantiation. */
- @FunctionalInterface
- public interface DelegateConfigure<T> {
- void configure(T delegate, Map<String, Object> stormConf, JsonNode params);
- }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java
deleted file mode 100644
index f7617205..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch;
-
-import com.google.common.io.Resources;
-import java.io.IOException;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import org.opensearch.client.opensearch.OpenSearchClient;
-import org.opensearch.client.opensearch.generic.Requests;
-import org.opensearch.client.opensearch.generic.Response;
-import org.opensearch.client.opensearch.indices.ExistsTemplateRequest;
-import org.slf4j.Logger;
-
-public class IndexCreation {
-
- public static synchronized void checkOrCreateIndex(
- OpenSearchClient client, String indexName, String boltType, Logger log)
- throws IOException {
- final boolean indexExists = client.indices().exists(req -> req.index(indexName)).value();
- log.info("Index '{}' exists? {}", indexName, indexExists);
- // there's a possible check-then-update race condition
- // createIndex intentionally catches and logs exceptions from OpenSearch
- if (!indexExists) {
- boolean created =
- IndexCreation.createIndex(client, indexName, boltType + ".mapping", log);
- log.info("Index '{}' created? {} using {}", indexName, created, boltType + ".mapping");
- }
- }
-
- public static synchronized void checkOrCreateIndexTemplate(
- OpenSearchClient client, String boltType, Logger log) throws IOException {
- final String templateName = boltType + "-template";
- final boolean templateExists =
- client.indices()
- .existsTemplate(ExistsTemplateRequest.of(r -> r.name(templateName)))
- .value();
- log.info("Template '{}' exists? {}", templateName, templateExists);
- // there's a possible check-then-update race condition
- // createTemplate intentionally catches and logs exceptions from OpenSearch
- if (!templateExists) {
- boolean created =
- IndexCreation.createTemplate(client, templateName, boltType + ".mapping", log);
- log.info("templateExists '{}' created? {}", templateName, created);
- }
- }
-
- private static boolean createTemplate(
- OpenSearchClient client, String templateName, String resourceName, Logger log) {
-
- try {
- final URL mapping =
- Thread.currentThread().getContextClassLoader().getResource(resourceName);
-
- final String jsonIndexConfiguration =
- Resources.toString(mapping, StandardCharsets.UTF_8);
-
- try (Response response =
- client.generic()
- .execute(
- Requests.builder()
- .endpoint("/_template/" + templateName)
- .method("PUT")
- .json(jsonIndexConfiguration)
- .build())) {
- int statusCode = response.getStatus();
- return statusCode == 200 || statusCode == 201;
- }
- } catch (Exception e) {
- log.warn("template '{}' not created", templateName, e);
- return false;
- }
- }
-
- private static boolean createIndex(
- OpenSearchClient client, String indexName, String resourceName, Logger log) {
-
- try {
- final URL mapping =
- Thread.currentThread().getContextClassLoader().getResource(resourceName);
-
- final String jsonIndexConfiguration =
- Resources.toString(mapping, StandardCharsets.UTF_8);
-
- try (Response response =
- client.generic()
- .execute(
- Requests.builder()
- .endpoint("/" + indexName)
- .method("PUT")
- .json(jsonIndexConfiguration)
- .build())) {
- int statusCode = response.getStatus();
- return statusCode == 200 || statusCode == 201;
- }
- } catch (Exception e) {
- log.warn("index '{}' not created", indexName, e);
- return false;
- }
- }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
deleted file mode 100644
index eeb47362..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
+++ /dev/null
@@ -1,516 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch;
-
-import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_CONNECT_TIMEOUT_MILLIS;
-import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_RESPONSE_TIMEOUT_MILLIS;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.regex.Pattern;
-import javax.net.ssl.SSLContext;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hc.client5.http.auth.AuthScope;
-import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
-import org.apache.hc.client5.http.config.ConnectionConfig;
-import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
-import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
-import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
-import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
-import org.apache.hc.core5.http.HttpHost;
-import org.apache.hc.core5.reactor.ssl.TlsDetails;
-import org.apache.hc.core5.ssl.SSLContextBuilder;
-import org.apache.hc.core5.util.Timeout;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.jetbrains.annotations.NotNull;
-import org.opensearch.client.json.jackson.JacksonJsonpMapper;
-import org.opensearch.client.opensearch.OpenSearchAsyncClient;
-import org.opensearch.client.opensearch.OpenSearchClient;
-import org.opensearch.client.opensearch.core.bulk.BulkOperation;
-import org.opensearch.client.transport.OpenSearchTransport;
-import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Utility class to instantiate an OpenSearch client and bulkprocessor based on the configuration.
- */
-public final class OpenSearchConnection {
-
- private static final Logger LOG = LoggerFactory.getLogger(OpenSearchConnection.class);
-
- @NotNull private final OpenSearchClient client;
-
- @NotNull private final AsyncBulkProcessor processor;
-
- @NotNull private final OpenSearchTransport transport;
-
- private OpenSearchConnection(
- @NotNull OpenSearchClient c,
- @NotNull AsyncBulkProcessor p,
- @NotNull OpenSearchTransport t) {
- client = c;
- processor = p;
- transport = t;
- }
-
- public OpenSearchClient getClient() {
- return client;
- }
-
- /**
- * Creates a standalone {@link OpenSearchClient}. Used by classes that need a client without a
- * bulk processor (e.g. spouts, filters). Callers are responsible for closing the returned
- * client's transport via {@code client._transport().close()}.
- */
- public static OpenSearchClient getClient(Map<String, Object> stormConf, String boltType) {
- return buildClientResources(stormConf, boltType).client();
- }
-
- /** Adds a single bulk operation to the internal processor. */
- public void addToProcessor(final BulkOperation operation) {
- processor.add(operation);
- }
-
- /**
- * Creates a connection with a default (no-op) listener. The values for bolt type are
- * [indexer,status,metrics].
- */
- public static OpenSearchConnection getConnection(
- Map<String, Object> stormConf, String boltType) {
- AsyncBulkProcessor.Listener listener =
- new AsyncBulkProcessor.Listener() {
- @Override
- public void afterBulk(
- long arg0,
- org.opensearch.client.opensearch.core.BulkRequest arg1,
- org.opensearch.client.opensearch.core.BulkResponse arg2) {}
-
- @Override
- public void afterBulk(
- long arg0,
- org.opensearch.client.opensearch.core.BulkRequest arg1,
- Throwable arg2) {}
-
- @Override
- public void beforeBulk(
- long arg0, org.opensearch.client.opensearch.core.BulkRequest arg1) {}
- };
- return getConnection(stormConf, boltType, listener);
- }
-
- public static OpenSearchConnection getConnection(
- Map<String, Object> stormConf, String boltType, AsyncBulkProcessor.Listener listener) {
-
- final String dottedType = boltType + ".";
-
- warnOnRemovedKeys(stormConf, dottedType);
-
- ClientResources cr = buildClientResources(stormConf, boltType);
-
- final String flushIntervalString =
- ConfUtils.getString(
- stormConf, Constants.PARAMPREFIX, dottedType, "flushInterval", "5s");
-
- final long flushIntervalMillis = parseTimeValueToMillis(flushIntervalString, 5000);
-
- final int bulkActions =
- ConfUtils.getInt(stormConf, Constants.PARAMPREFIX, dottedType, "bulkActions", 50);
-
- final int concurrentRequests =
- ConfUtils.getInt(
- stormConf, Constants.PARAMPREFIX, dottedType, "concurrentRequests", 1);
-
- AsyncBulkProcessor bulkProcessor = null;
- try {
- bulkProcessor =
- new AsyncBulkProcessor.Builder(
- new OpenSearchAsyncClient(cr.transport()), listener)
- .setBulkActions(bulkActions)
- .setFlushIntervalMillis(flushIntervalMillis)
- .setConcurrentRequests(concurrentRequests)
- .build();
-
- return new OpenSearchConnection(cr.client(), bulkProcessor, cr.transport());
- } catch (Exception e) {
- if (bulkProcessor != null) {
- try {
- bulkProcessor.close();
- } catch (Exception suppressed) {
- e.addSuppressed(suppressed);
- }
- }
- try {
- cr.transport().close();
- } catch (IOException suppressed) {
- e.addSuppressed(suppressed);
- }
- throw e;
- }
- }
-
- private final AtomicBoolean isClosed = new AtomicBoolean(false);
-
- public void close() {
-
- if (!isClosed.compareAndSet(false, true)) {
- LOG.warn("Tried to close an already closed connection!");
- return;
- }
-
- LOG.debug("Start closing the OpenSearch connection");
-
- // First, close the BulkProcessor ensuring pending actions are flushed
- try {
- boolean success = processor.awaitClose(60, TimeUnit.SECONDS);
- if (!success) {
- throw new RuntimeException(
- "Failed to flush pending actions when closing BulkProcessor");
- }
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
-
- // Now close the transport (also shuts down the underlying HTTP client)
- try {
- transport.close();
- } catch (IOException e) {
- LOG.trace("Transport threw IO exception on close.");
- }
- }
-
- /**
- * Extracts the document ID from a {@link BulkOperation} regardless of its type (index, create,
- * delete, update).
- */
- public static String getBulkOperationId(BulkOperation op) {
- if (op.isIndex()) {
- return op.index().id();
- }
- if (op.isCreate()) {
- return op.create().id();
- }
- if (op.isDelete()) {
- return op.delete().id();
- }
- if (op.isUpdate()) {
- return op.update().id();
- }
- return null;
- }
-
- /**
- * Returns the scopes the Basic credentials are registered for: the scheme, host and port of
- * each configured address. A request to any other address does not receive them.
- */
- static List<AuthScope> credentialScopes(List<HttpHost> hosts) {
- final Set<AuthScope> scopes = new LinkedHashSet<>();
- for (HttpHost host : hosts) {
- scopes.add(new AuthScope(host));
- }
- return new ArrayList<>(scopes);
- }
-
- /**
- * Returns the addresses which use plain http and are not a loopback address, i.e. those the
- * Basic credentials would be sent to in the clear over the network.
- */
- static List<HttpHost> plainHttpHosts(List<HttpHost> hosts) {
- final List<HttpHost> plain = new ArrayList<>();
- for (HttpHost host : hosts) {
- if ("http".equalsIgnoreCase(host.getSchemeName()) && !isLoopback(host.getHostName())) {
- plain.add(host);
- }
- }
- return plain;
- }
-
- private static final Pattern IPV4_LOOPBACK = Pattern.compile("127(\\.\\d{1,3}){3}");
-
- /** Whether the host is localhost or a loopback IP literal. No name is resolved. */
- static boolean isLoopback(String hostname) {
- String host = StringUtils.strip(hostname.toLowerCase(Locale.ROOT), "[]");
- return host.equals("localhost")
- || IPV4_LOOPBACK.matcher(host).matches()
- || host.equals("::1")
- || host.equals("0:0:0:0:0:0:0:1");
- }
-
- private static void warnAboutPlainHttp(String boltType, List<HttpHost> hosts) {
- final List<HttpHost> plain = plainHttpHosts(hosts);
- if (!plain.isEmpty()) {
- LOG.warn(
- "OpenSearch credentials are configured for {} but the addresses {} use plain "
- + "http, the credentials are sent unencrypted. Give the addresses an "
- + "https:// scheme.",
- boltType,
- plain);
- }
- }
-
- // internal helpers
- private record ClientResources(OpenSearchClient client, OpenSearchTransport transport) {}
-
- /**
- * Logs a WARN for legacy configuration keys that are no longer honoured by this module, so that
- * users migrating from {@code external/opensearch} notice silently-dropped tuning. See the
- * module README for the full list of differences.
- */
- private static void warnOnRemovedKeys(Map<String, Object> stormConf, String dottedType) {
- final String responseBufferKey = Constants.PARAMPREFIX + dottedType + "responseBufferSize";
- if (stormConf.containsKey(responseBufferKey)) {
- LOG.warn(
- "Configuration key '{}' is set but no longer supported by the opensearch-java module. "
- + "The HC5-based async transport does not expose an equivalent per-request "
- + "heap-buffer override. The setting is ignored — remove it from your "
- + "configuration. See external/opensearch-java/README.md for details.",
- responseBufferKey);
- }
-
- final String sniffKey = Constants.PARAMPREFIX + dottedType + "sniff";
- if (stormConf.containsKey(sniffKey)) {
- LOG.warn(
- "Configuration key '{}' is set but no longer supported by the opensearch-java module. "
- + "The OpenSearch Java Client 3.x does not ship a Sniffer equivalent, so "
- + "automatic node discovery is not available. Keep the 'addresses' list up to "
- + "date manually or put a load balancer in front of the cluster. "
- + "See external/opensearch-java/README.md for details.",
- sniffKey);
- }
- }
-
- private static ClientResources buildClientResources(
- Map<String, Object> stormConf, String boltType) {
-
- final String dottedType = boltType + ".";
-
- final List<HttpHost> hosts = new ArrayList<>();
-
- final List<String> confighosts =
- ConfUtils.loadListFromConf(
- Constants.PARAMPREFIX, dottedType, "addresses", stormConf);
-
- // find ; separated values and tokenise as multiple addresses
- // e.g. opensearch1:9200; opensearch2:9200
- if (confighosts.size() == 1) {
- String input = confighosts.get(0);
- confighosts.clear();
- confighosts.addAll(Arrays.asList(input.split(" *; *")));
- }
-
- for (String host : confighosts) {
- // no port specified? use default one
- int port = 9200;
- String scheme = "http";
- // no scheme specified? use http
- if (!host.startsWith(scheme)) {
- host = "http://" + host;
- }
- URI uri = URI.create(host);
- if (uri.getHost() == null) {
- throw new RuntimeException("host undefined " + host);
- }
- if (uri.getPort() != -1) {
- port = uri.getPort();
- }
- if (uri.getScheme() != null) {
- scheme = uri.getScheme();
- }
- // HC5: constructor is (scheme, hostname, port) — not (hostname, port, scheme)
- hosts.add(new HttpHost(scheme, uri.getHost(), port));
- }
-
- LOG.info(
- "OpenSearch {} transport configured with {} host(s): {}",
- boltType,
- hosts.size(),
- hosts);
-
- // authentication via user / password
- final String user =
- ConfUtils.getString(stormConf, Constants.PARAMPREFIX, dottedType, "user");
- final String password =
- ConfUtils.getString(stormConf, Constants.PARAMPREFIX, dottedType, "password");
-
- final String proxyhost =
- ConfUtils.getString(stormConf, Constants.PARAMPREFIX, dottedType, "proxy.host");
-
- final int proxyport =
- ConfUtils.getInt(stormConf, Constants.PARAMPREFIX, dottedType, "proxy.port", -1);
-
- final String proxyscheme =
- ConfUtils.getString(
- stormConf, Constants.PARAMPREFIX, dottedType, "proxy.scheme", "http");
-
- final boolean disableTlsValidation =
- ConfUtils.getBoolean(
- stormConf, Constants.PARAMPREFIX, "", "disable.tls.validation", false);
-
- final boolean needsUser = StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password);
- final boolean needsProxy = StringUtils.isNotBlank(proxyhost) && proxyport != -1;
-
- // Defaults from ApacheHttpClient5TransportBuilder (same as the former RestClientBuilder)
- final int connectTimeout =
- ConfUtils.getInt(
- stormConf,
- Constants.PARAMPREFIX,
- dottedType,
- "connect.timeout",
- DEFAULT_CONNECT_TIMEOUT_MILLIS);
- final int socketTimeout =
- ConfUtils.getInt(
- stormConf,
- Constants.PARAMPREFIX,
- dottedType,
- "socket.timeout",
- DEFAULT_RESPONSE_TIMEOUT_MILLIS);
-
- final boolean compression =
- ConfUtils.getBoolean(
- stormConf, Constants.PARAMPREFIX, dottedType, "compression", false);
-
- final ApacheHttpClient5TransportBuilder builder =
- ApacheHttpClient5TransportBuilder.builder(hosts.toArray(new HttpHost[0]))
- .setMapper(new JacksonJsonpMapper());
-
- // Timeouts via ConnectionConfig on the builder's internal connection manager
- builder.setConnectionConfigCallback(
- connConfigBuilder ->
- connConfigBuilder
- .setConnectTimeout(Timeout.ofMilliseconds(connectTimeout))
- .setSocketTimeout(Timeout.ofMilliseconds(socketTimeout)));
-
- if (needsUser) {
- warnAboutPlainHttp(boltType, hosts);
- }
- if (disableTlsValidation) {
- LOG.warn(
- "opensearch.disable.tls.validation is set: the certificates and host names of "
- + "the OpenSearch nodes used for {} are not checked",
- boltType);
- }
-
- // Auth, proxy, and/or trust-all SSL via HttpClient customisation
- if (needsUser || needsProxy || disableTlsValidation) {
- builder.setHttpClientConfigCallback(
- httpClientBuilder -> {
- // hc.client5 auth: password is char[], AuthScope(host, port)
- if (needsUser) {
- final BasicCredentialsProvider credentialsProvider =
- new BasicCredentialsProvider();
- final UsernamePasswordCredentials credentials =
- new UsernamePasswordCredentials(user, password.toCharArray());
- for (AuthScope scope : credentialScopes(hosts)) {
- credentialsProvider.setCredentials(scope, credentials);
- }
- httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
- }
- // hc.client5 proxy: HttpHost(scheme, host, port)
- if (needsProxy) {
- httpClientBuilder.setProxy(
- new HttpHost(proxyscheme, proxyhost, proxyport));
- }
- // Custom connection manager overrides the builder's internal one,
- // so timeouts and TlsDetailsFactory must be replicated here
- if (disableTlsValidation) {
- try {
- final SSLContext sslContext =
- SSLContextBuilder.create()
- .loadTrustMaterial((chain, authType) -> true)
- .build();
- httpClientBuilder.setConnectionManager(
- PoolingAsyncClientConnectionManagerBuilder.create()
- .setTlsStrategy(
- ClientTlsStrategyBuilder.create()
- .setSslContext(sslContext)
- .setHostnameVerifier(
- NoopHostnameVerifier
- .INSTANCE)
- // HTTP/2 ALPN negotiation
- .setTlsDetailsFactory(
- sslEngine ->
- new TlsDetails(
- sslEngine
- .getSession(),
- sslEngine
- .getApplicationProtocol()))
- .build())
- .setDefaultConnectionConfig(
- ConnectionConfig.custom()
- .setConnectTimeout(
- Timeout.ofMilliseconds(
- connectTimeout))
- .setSocketTimeout(
- Timeout.ofMilliseconds(
- socketTimeout))
- .build())
- .build());
- } catch (Exception e) {
- throw new RuntimeException("Failed to disable TLS validation", e);
- }
- }
- return httpClientBuilder;
- });
- }
-
- // Compression: first-class builder method, not a request interceptor
- builder.setCompressionEnabled(compression);
-
- final OpenSearchTransport transport = builder.build();
- final OpenSearchClient openSearchClient = new OpenSearchClient(transport);
-
- return new ClientResources(openSearchClient, transport);
- }
-
- /**
- * Parses a time value string (e.g. "5s", "500ms", "1m") into milliseconds.
- *
- * @param value the string to parse
- * @param defaultMillis the default if parsing fails
- * @return milliseconds
- */
- static long parseTimeValueToMillis(String value, long defaultMillis) {
- if (value == null || value.isBlank()) {
- return defaultMillis;
- }
- value = value.trim();
- try {
- if (value.endsWith("ms")) {
- return Long.parseLong(value.substring(0, value.length() - 2));
- } else if (value.endsWith("s")) {
- return Long.parseLong(value.substring(0, value.length() - 1)) * 1000;
- } else if (value.endsWith("m")) {
- return Long.parseLong(value.substring(0, value.length() - 1)) * 60000;
- } else {
- return Long.parseLong(value);
- }
- } catch (NumberFormatException e) {
- LOG.warn("Could not parse time value '{}', using default {}ms", value, defaultMillis);
- return defaultMillis;
- }
- }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java
deleted file mode 100644
index 0e8574fb..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java
+++ /dev/null
@@ -1,340 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch;
-
-import com.github.benmanes.caffeine.cache.Cache;
-import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.RemovalCause;
-import com.github.benmanes.caffeine.cache.Ticker;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
-import org.apache.storm.tuple.Tuple;
-import org.apache.stormcrawler.metrics.ScopedCounter;
-import org.jetbrains.annotations.Nullable;
-// opensearch-java: uses typed BulkRequest/BulkResponse, not legacy REST equivalents
-import org.opensearch.client.opensearch.core.BulkRequest;
-import org.opensearch.client.opensearch.core.BulkResponse;
-import org.slf4j.Logger;
-
-/**
- * Thread-safe cache that tracks in-flight tuples awaiting bulk acknowledgment from OpenSearch.
- * Provides shared logic for processing bulk responses and failing tuples on error, used by
- * IndexerBolt, DeletionBolt, and StatusUpdaterBolt.
- */
-public class WaitAckCache {
-
- /** Callback invoked for each tuple when processing a successful bulk response. */
- @FunctionalInterface
- public interface TupleAction {
- void handle(String id, Tuple tuple, BulkItemResponseToFailedFlag selected);
- }
-
- private final Cache<String, List<Tuple>> cache;
- private final ReentrantLock lock = new ReentrantLock(true);
- private final Logger log;
- private final Consumer<Tuple> onEviction;
-
- /** Creates a cache with a fixed 60-second expiry. */
- public WaitAckCache(Logger log, Consumer<Tuple> onEviction) {
- this(Caffeine.newBuilder().expireAfterWrite(60, TimeUnit.SECONDS), log, onEviction);
- }
-
- /**
- * Creates a cache using a {@link Caffeine} spec string (e.g. {@code "expireAfterWrite=300s"}),
- * typically driven by {@code topology.message.timeout.secs}.
- */
- public WaitAckCache(String cacheSpec, Logger log, Consumer<Tuple> onEviction) {
- this(Caffeine.from(cacheSpec), log, onEviction);
- }
-
- /** Creates a cache with a custom ticker for deterministic time control in tests. */
- WaitAckCache(String cacheSpec, Logger log, Consumer<Tuple> onEviction, Ticker ticker) {
- this(Caffeine.from(cacheSpec).ticker(ticker).executor(Runnable::run), log, onEviction);
- }
-
- private WaitAckCache(Caffeine<Object, Object> builder, Logger log, Consumer<Tuple> onEviction) {
- this.log = log;
- this.onEviction = onEviction;
- this.cache =
- builder.<String, List<Tuple>>removalListener(
- (String key, List<Tuple> value, RemovalCause cause) -> {
- if (!cause.wasEvicted()) {
- return;
- }
- if (value != null) {
- log.error(
- "Purged from waitAck {} with {} values",
- key,
- value.size());
- for (Tuple t : value) {
- onEviction.accept(t);
- }
- } else {
- log.error("Purged from waitAck {} with no values", key);
- }
- })
- .build();
- }
-
- /** Returns the approximate number of entries in this cache. */
- public long estimatedSize() {
- return cache.estimatedSize();
- }
-
- /** Adds a tuple to the cache under the given document ID, creating the list if needed. */
- public void addTuple(String docID, Tuple tuple) {
- lock.lock();
- try {
- List<Tuple> tt = cache.get(docID, k -> new LinkedList<>());
- tt.add(tuple);
- if (log.isDebugEnabled()) {
- String url = (String) tuple.getValueByField("url");
- log.debug("Added to waitAck {} with ID {} total {}", url, docID, tt.size());
- }
- } finally {
- lock.unlock();
- }
- }
-
- /** Returns true if the cache contains an entry for the given document ID. */
- public boolean contains(String docID) {
- lock.lock();
- try {
- return cache.getIfPresent(docID) != null;
- } finally {
- lock.unlock();
- }
- }
-
- /** Forces pending cache maintenance, triggering eviction listeners for expired entries. */
- public void cleanUp() {
- cache.cleanUp();
- }
-
- /** Fails all remaining tuples in the cache and invalidates all entries. */
- public void shutdown() {
- lock.lock();
- try {
- Map<String, List<Tuple>> remaining = cache.asMap();
- for (var entry : remaining.entrySet()) {
- log.warn(
- "Shutdown: failing {} tuple(s) for ID {}",
- entry.getValue().size(),
- entry.getKey());
- for (Tuple t : entry.getValue()) {
- onEviction.accept(t);
- }
- }
- cache.invalidateAll();
- } finally {
- lock.unlock();
- }
- }
-
- /** Invalidates a single cache entry. */
- public void invalidate(String docID) {
- lock.lock();
- try {
- cache.invalidate(docID);
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * Processes a successful bulk response: classifies each item (conflict vs failure), retrieves
- * cached tuples, selects the best response per document ID, and invokes the action for each
- * tuple.
- *
- * @param conflictCounter optional metric counter; if non-null, increments "doc_conflicts" scope
- * for each conflict
- */
- public void processBulkResponse(
- BulkResponse response,
- long executionId,
- @Nullable ScopedCounter conflictCounter,
- TupleAction action) {
-
- // opensearch-java: items() returns List<BulkResponseItem>; status() returns int
- var idsToBulkItems =
- response.items().stream()
- .map(
- bir -> {
- var error = bir.error();
- boolean failed = false;
- if (error != null) {
- // opensearch-java: int status code, not RestStatus enum
- if (bir.status() == 409) {
- if (conflictCounter != null) {
- conflictCounter.scope("doc_conflicts").incrBy(1);
- }
- log.debug("Doc conflict ID {}", bir.id());
- } else {
- log.error(
- "Bulk item failure ID {}: {}",
- bir.id(),
- error.reason() != null
- ? error.reason()
- : error.type());
- failed = true;
- }
- }
- return new BulkItemResponseToFailedFlag(bir, failed);
- })
- .collect(
- // https://github.com/apache/stormcrawler/issues/832
- Collectors.groupingBy(
- BulkItemResponseToFailedFlag::id,
- Collectors.toUnmodifiableList()));
-
- Map<String, List<Tuple>> presentTuples;
- long estimatedSize;
- Set<String> debugInfo = null;
- lock.lock();
- try {
- presentTuples = cache.getAllPresent(idsToBulkItems.keySet());
- if (!presentTuples.isEmpty()) {
- cache.invalidateAll(presentTuples.keySet());
- }
- estimatedSize = cache.estimatedSize();
- if (log.isDebugEnabled() && estimatedSize > 0L) {
- debugInfo = new HashSet<>(cache.asMap().keySet());
- }
- } finally {
- lock.unlock();
- }
-
- int ackCount = 0;
- int failureCount = 0;
-
- for (var entry : presentTuples.entrySet()) {
- final var id = entry.getKey();
- final var tuples = entry.getValue();
- final var bulkItems = idsToBulkItems.get(id);
-
- BulkItemResponseToFailedFlag selected = selectBest(bulkItems, id);
-
- if (tuples != null) {
- log.debug("Found {} tuple(s) for ID {}", tuples.size(), id);
- for (Tuple t : tuples) {
- if (selected.failed()) {
- failureCount++;
- } else {
- ackCount++;
- }
- action.handle(id, t, selected);
- }
- } else {
- log.warn("Could not find unacked tuples for {}", id);
- }
- }
-
- log.info(
- "Bulk response [{}] : items {}, waitAck {}, acked {}, failed {}",
- executionId,
- idsToBulkItems.size(),
- estimatedSize,
- ackCount,
- failureCount);
-
- if (debugInfo != null) {
- for (String k : debugInfo) {
- log.debug("Still in wait ack after bulk response [{}] => {}", executionId, k);
- }
- }
- }
-
- /**
- * Processes a failed bulk request by failing all associated tuples.
- *
- * @param failAction callback applied to each tuple that must be failed
- */
- public void processFailedBulk(
- BulkRequest request, long executionId, Throwable failure, Consumer<Tuple> failAction) {
-
- log.error("Exception with bulk {} - failing the whole lot ", executionId, failure);
-
- // opensearch-java: operations() + getBulkOperationId replaces
- // legacy requests() + DocWriteRequest::id
- final var failedIds =
- request.operations().stream()
- .map(OpenSearchConnection::getBulkOperationId)
- .filter(Objects::nonNull)
- .collect(Collectors.toUnmodifiableSet());
-
- Map<String, List<Tuple>> failedTupleLists;
- lock.lock();
- try {
- failedTupleLists = cache.getAllPresent(failedIds);
- if (!failedTupleLists.isEmpty()) {
- cache.invalidateAll(failedTupleLists.keySet());
- }
- } finally {
- lock.unlock();
- }
-
- for (var id : failedIds) {
- var tuples = failedTupleLists.get(id);
- if (tuples != null) {
- log.debug("Failed {} tuple(s) for ID {}", tuples.size(), id);
- for (Tuple t : tuples) {
- failAction.accept(t);
- }
- } else {
- log.warn("Could not find unacked tuple for {}", id);
- }
- }
- }
-
- /**
- * Selects the best response when there are multiple bulk items for the same document ID.
- * Prefers non-failed responses; warns when there is a mix of success and failure. If all items
- * are failed, returns the first one (no warning logged since there is no ambiguity).
- */
- private BulkItemResponseToFailedFlag selectBest(
- List<BulkItemResponseToFailedFlag> items, String id) {
- if (items.size() == 1) {
- return items.get(0);
- }
-
- BulkItemResponseToFailedFlag best = items.get(0);
- int failedCount = 0;
- for (var item : items) {
- if (item.failed()) {
- failedCount++;
- } else {
- best = item;
- }
- }
- if (failedCount > 0 && failedCount < items.size()) {
- log.warn(
- "The id {} would result in an ack and a failure."
- + " Using only the ack for processing.",
- id);
- }
- return best;
- }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
deleted file mode 100644
index f77662ea..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.bolt;
-
-import java.lang.invoke.MethodHandles;
-import java.util.Map;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.topology.OutputFieldsDeclarer;
-import org.apache.storm.topology.base.BaseRichBolt;
-import org.apache.storm.tuple.Tuple;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
-import org.apache.stormcrawler.metrics.CrawlerMetrics;
-import org.apache.stormcrawler.opensearch.AsyncBulkProcessor;
-import org.apache.stormcrawler.opensearch.OpenSearchConnection;
-import org.apache.stormcrawler.opensearch.WaitAckCache;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.opensearch.core.BulkRequest;
-import org.opensearch.client.opensearch.core.BulkResponse;
-import org.opensearch.client.opensearch.core.bulk.BulkOperation;
-import org.slf4j.LoggerFactory;
-
-/**
- * Deletes documents in OpenSearch. This should be connected to the StatusUpdaterBolt via the
- * 'deletion' stream and will remove the documents with a status of ERROR. Note that this component
- * will also try to delete documents even though they were never indexed and it currently won't
- * delete documents which were indexed under the canonical URL.
- */
-public class DeletionBolt extends BaseRichBolt implements AsyncBulkProcessor.Listener {
-
- static final org.slf4j.Logger LOG =
- LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final String BOLT_TYPE = "indexer";
-
- private OutputCollector _collector;
-
- private String indexName;
-
- private OpenSearchConnection connection;
-
- private WaitAckCache waitAck;
-
- private String docIdMetadataKey;
-
- public DeletionBolt() {}
-
- /** Sets the index name instead of taking it from the configuration. * */
- public DeletionBolt(String indexName) {
- this.indexName = indexName;
- }
-
- @Override
- public void prepare(
- Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
- _collector = collector;
- if (indexName == null) {
- indexName = ConfUtils.getString(conf, IndexerBolt.OSIndexNameParamName, "content");
- }
-
- docIdMetadataKey =
- ConfUtils.getString(conf, AbstractIndexerBolt.DOC_ID_METADATA_PARAM_NAME);
-
- try {
- connection = OpenSearchConnection.getConnection(conf, BOLT_TYPE, this);
- } catch (Exception e1) {
- LOG.error("Can't connect to OpenSearch", e1);
- throw new RuntimeException(e1);
- }
-
- waitAck = new WaitAckCache(LOG, _collector::fail);
- CrawlerMetrics.registerGauge(context, conf, "waitAck", waitAck::estimatedSize, 10);
- }
-
- @Override
- public void cleanup() {
- waitAck.shutdown();
- if (connection != null) {
- connection.close();
- }
- }
-
- @Override
- public void execute(Tuple tuple) {
- String url = tuple.getStringByField("url");
- Metadata metadata = (Metadata) tuple.getValueByField("metadata");
-
- // keep it simple for now and ignore cases where the canonical URL was
- // used
-
- final String docID = getDocumentID(metadata, url);
- final String targetIndex = getIndexName(metadata);
- BulkOperation op = BulkOperation.of(b -> b.delete(d -> d.index(targetIndex).id(docID)));
-
- waitAck.addTuple(docID, tuple);
-
- connection.addToProcessor(op);
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer arg0) {
- // none
- }
-
- /**
- * Must be overridden for implementing custom index names based on some metadata information By
- * Default, indexName coming from config is used
- */
- protected String getIndexName(Metadata m) {
- return indexName;
- }
-
- /**
- * Get the document id. Uses the SHA-256 digest of the value found in the metadata for the key
- * configured with {@link AbstractIndexerBolt#DOC_ID_METADATA_PARAM_NAME} if present, so that
- * the ID matches the one generated by the indexer for the same document.
- *
- * @param metadata The {@link Metadata}.
- * @param url The normalised url.
- * @return Return the SHA-256 digest of the value found in the metadata, or of the normalised
- * url if not present.
- */
- protected String getDocumentID(Metadata metadata, String url) {
- String source = url;
- if (docIdMetadataKey != null) {
- final String fromMetadata =
- StringUtils.trimToNull(metadata.getFirstValue(docIdMetadataKey));
- if (fromMetadata != null) {
- source = fromMetadata;
- }
- }
- return org.apache.commons.codec.digest.DigestUtils.sha256Hex(source);
- }
-
- @Override
- public void beforeBulk(long executionId, BulkRequest request) {}
-
- @Override
- public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
- waitAck.processBulkResponse(
- response,
- executionId,
- null,
- (id, t, selected) -> {
- if (!selected.failed()) {
- _collector.ack(t);
- } else {
- String url = (String) t.getValueByField("url");
- LOG.error(
- "update ID {}, URL {}, failure: {}",
- id,
- url,
- selected.getFailure());
- _collector.fail(t);
- }
- });
- }
-
- @Override
- public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
- waitAck.processFailedBulk(request, executionId, failure, _collector::fail);
- }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
deleted file mode 100644
index f7a44185..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.bolt;
-
-import static org.apache.stormcrawler.Constants.StatusStreamName;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.tuple.Values;
-import org.apache.stormcrawler.Constants;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
-import org.apache.stormcrawler.metrics.CrawlerMetrics;
-import org.apache.stormcrawler.metrics.ScopedCounter;
-import org.apache.stormcrawler.metrics.ScopedReducedMetric;
-import org.apache.stormcrawler.opensearch.AsyncBulkProcessor;
-import org.apache.stormcrawler.opensearch.IndexCreation;
-import org.apache.stormcrawler.opensearch.OpenSearchConnection;
-import org.apache.stormcrawler.opensearch.WaitAckCache;
-import org.apache.stormcrawler.persistence.Status;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.opensearch.core.BulkRequest;
-import org.opensearch.client.opensearch.core.BulkResponse;
-import org.opensearch.client.opensearch.core.bulk.BulkOperation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Sends documents to opensearch. Indexes all the fields from the tuples or a Map
- * <String,Object> from a named field.
- */
-public class IndexerBolt extends AbstractIndexerBolt implements AsyncBulkProcessor.Listener {
-
- private static final Logger LOG = LoggerFactory.getLogger(IndexerBolt.class);
-
- private static final String OSBoltType = "indexer";
-
- static final String OSIndexNameParamName =
- org.apache.stormcrawler.opensearch.Constants.PARAMPREFIX + OSBoltType + ".index.name";
- private static final String OSCreateParamName =
- org.apache.stormcrawler.opensearch.Constants.PARAMPREFIX + OSBoltType + ".create";
- private static final String OSIndexPipelineParamName =
- org.apache.stormcrawler.opensearch.Constants.PARAMPREFIX + OSBoltType + ".pipeline";
-
- private OutputCollector _collector;
-
- private String indexName;
-
- private String pipeline;
-
- // whether the document will be created only if it does not exist or
- // overwritten
- private boolean create = false;
-
- private ScopedCounter eventCounter;
-
- private OpenSearchConnection connection;
-
- private ScopedReducedMetric perSecMetrics;
-
- private WaitAckCache waitAck;
-
- public IndexerBolt() {}
-
- /** Sets the index name instead of taking it from the configuration. * */
- public IndexerBolt(String indexName) {
- this.indexName = indexName;
- }
-
- @Override
- public void prepare(
- Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
- super.prepare(conf, context, collector);
- _collector = collector;
- if (indexName == null) {
- indexName = ConfUtils.getString(conf, IndexerBolt.OSIndexNameParamName, "content");
- }
-
- create = ConfUtils.getBoolean(conf, IndexerBolt.OSCreateParamName, false);
- pipeline = ConfUtils.getString(conf, IndexerBolt.OSIndexPipelineParamName);
-
- try {
- connection = OpenSearchConnection.getConnection(conf, OSBoltType, this);
- } catch (Exception e1) {
- LOG.error("Can't connect to OpenSearch", e1);
- throw new RuntimeException(e1);
- }
-
- this.eventCounter = CrawlerMetrics.registerCounter(context, conf, "OpensearchIndexer", 10);
-
- this.perSecMetrics =
- CrawlerMetrics.registerPerSecMetric(context, conf, "Indexer_average_persec", 10);
-
- waitAck = new WaitAckCache(LOG, _collector::fail);
- CrawlerMetrics.registerGauge(context, conf, "waitAck", waitAck::estimatedSize, 10);
-
- // use the default status schema if none has been specified
- try {
- IndexCreation.checkOrCreateIndex(connection.getClient(), indexName, OSBoltType, LOG);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void cleanup() {
- waitAck.shutdown();
- if (connection != null) {
- connection.close();
- }
- }
-
- @Override
- public void execute(Tuple tuple) {
-
- final String url = tuple.getStringByField("url");
-
- // Distinguish the value used for indexing
- // from the one used for the status
- final String normalisedurl = valueForURL(tuple);
-
- LOG.info("Indexing {} as {}", url, normalisedurl);
-
- final Metadata metadata = (Metadata) tuple.getValueByField("metadata");
-
- if (!filterDocument(metadata)) {
- LOG.info("Filtered {}", url);
- eventCounter.scope("Filtered").incrBy(1);
- // treat it as successfully processed even if
- // we do not index it
- _collector.emit(StatusStreamName, tuple, new Values(url, metadata, Status.FETCHED));
- _collector.ack(tuple);
- return;
- }
-
- final String docID = getDocumentID(metadata, normalisedurl);
-
- try {
- final Map<String, Object> source = new HashMap<>();
-
- // display text of the document?
- if (StringUtils.isNotBlank(fieldNameForText())) {
- final String text = trimText(tuple.getStringByField("text"));
- if (!ignoreEmptyFields() || StringUtils.isNotBlank(text)) {
- source.put(fieldNameForText(), trimText(text));
- }
- }
-
- // send URL as field?
- if (StringUtils.isNotBlank(fieldNameForURL())) {
- source.put(fieldNameForURL(), normalisedurl);
- }
-
- // which metadata to display?
- final Map<String, String[]> keyVals = filterMetadata(metadata);
-
- for (Entry<String, String[]> entry : keyVals.entrySet()) {
- if (entry.getValue().length == 1) {
- final String value = entry.getValue()[0];
- if (!ignoreEmptyFields() || StringUtils.isNotBlank(value)) {
- source.put(entry.getKey(), value);
- }
- } else if (entry.getValue().length > 1) {
- source.put(entry.getKey(), Arrays.asList(entry.getValue()));
- }
- }
-
- final String targetIndex = getIndexName(metadata);
- final BulkOperation op;
- if (create) {
- op =
- BulkOperation.of(
- b ->
- b.create(
- c -> {
- c.index(targetIndex).id(docID).document(source);
- if (pipeline != null) {
- c.pipeline(pipeline);
- }
- return c;
- }));
- } else {
- op =
- BulkOperation.of(
- b ->
- b.index(
- idx -> {
- idx.index(targetIndex)
- .id(docID)
- .document(source);
- if (pipeline != null) {
- idx.pipeline(pipeline);
- }
- return idx;
- }));
- }
-
- waitAck.addTuple(docID, tuple);
-
- connection.addToProcessor(op);
-
- eventCounter.scope("Indexed").incrBy(1);
- perSecMetrics.scope("Indexed").update(1);
- } catch (Exception e) {
- LOG.error("Error building document for OpenSearch", e);
- // do not send to status stream so that it gets replayed
- _collector.fail(tuple);
-
- waitAck.invalidate(docID);
- }
- }
-
- /**
- * Must be overridden for implementing custom index names based on some metadata information By
- * Default, indexName coming from config is used
- */
- protected String getIndexName(Metadata m) {
- return indexName;
- }
-
- @Override
- public void beforeBulk(long executionId, BulkRequest request) {
- eventCounter.scope("bulks_sent").incrBy(1);
- }
-
- @Override
- public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
- eventCounter.scope("bulks_received").incrBy(1);
- eventCounter.scope("bulk_msec").incrBy(response.took());
-
- waitAck.processBulkResponse(
- response,
- executionId,
- eventCounter,
- (id, t, selected) -> {
- String url = (String) t.getValueByField("url");
- Metadata metadata = (Metadata) t.getValueByField("metadata");
-
- if (!selected.failed()) {
- _collector.emit(
- StatusStreamName, t, new Values(url, metadata, Status.FETCHED));
- _collector.ack(t);
- } else {
- var failure = selected.getFailure();
- LOG.error("update ID {}, URL {}, failure: {}", id, url, failure);
- // there is something wrong with the content we should
- // treat it as an ERROR
- if (selected.getStatus() == 400) {
- metadata.setValue(Constants.STATUS_ERROR_SOURCE, "OpenSearch indexing");
- metadata.setValue(Constants.STATUS_ERROR_MESSAGE, "invalid content");
- _collector.emit(
- StatusStreamName, t, new Values(url, metadata, Status.ERROR));
- _collector.ack(t);
- LOG.debug("Acked {} with ID {}", url, id);
- } else {
- _collector.fail(t);
- LOG.debug("Failed {} with ID {}", url, id);
- }
- }
- });
- }
-
- @Override
- public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
- eventCounter.scope("bulks_received").incrBy(1);
- waitAck.processFailedBulk(
- request,
- executionId,
- failure,
- t -> {
- eventCounter.scope("failed").incrBy(1);
- _collector.fail(t);
- });
- }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
deleted file mode 100644
index 49ee7f0c..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.filtering;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import java.net.URL;
-import java.util.Map;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.filtering.URLFilter;
-import org.apache.stormcrawler.opensearch.DelegateRefresher;
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-
-/**
- * Wraps a URLFilter whose resources are in a JSON file that can be stored in OpenSearch. The
- * benefit of doing this is that the resources can be refreshed automatically and modified without
- * having to recompile the jar and restart the topology. The connection to OpenSearch is done via
- * the config and uses a new bolt type 'config'.
- *
- * <p>The configuration of the delegate is done in the urlfilters.json as usual.
- *
- * <pre>
- * {
- * "class": "org.apache.stormcrawler.opensearch.filtering.JSONURLFilterWrapper",
- * "name": "OSFastURLFilter",
- * "params": {
- * "refresh": "60",
- * "delegate": {
- * "class": "org.apache.stormcrawler.filtering.regex.FastURLFilter",
- * "params": {
- * "file": "fast.urlfilter.json"
- * }
- * }
- * }
- * }
- * </pre>
- *
- * The resource file can be pushed to OpenSearch with
- *
- * <pre>
- * curl -XPUT 'localhost:9200/config/config/fast.urlfilter.json?pretty' -H 'Content-Type: application/json' -d @fast.urlfilter.json
- * </pre>
- */
-public class JSONURLFilterWrapper extends URLFilter {
-
- private DelegateRefresher<URLFilter> refresher;
-
- public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode filterParams) {
- refresher =
- new DelegateRefresher<>(
- URLFilter.class, stormConf, filterParams, URLFilter::configure);
- }
-
- @Override
- public @Nullable String filter(
- @Nullable URL sourceUrl,
- @Nullable Metadata sourceMetadata,
- @NotNull String urlToFilter) {
- return refresher.getDelegate().filter(sourceUrl, sourceMetadata, urlToFilter);
- }
-
- @Override
- public void cleanup() {
- if (refresher != null) {
- refresher.cleanup();
- }
- }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
deleted file mode 100644
index c46a2b73..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.metrics;
-
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Map.Entry;
-import org.apache.storm.metric.api.IMetricsConsumer;
-import org.apache.storm.task.IErrorReporter;
-import org.apache.storm.task.TopologyContext;
-import org.apache.stormcrawler.opensearch.IndexCreation;
-import org.apache.stormcrawler.opensearch.OpenSearchConnection;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.opensearch.core.bulk.BulkOperation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Sends metrics to an OpenSearch index. The OpenSearch details are set in the configuration; an
- * optional argument sets a date format to append to the index name.
- *
- * <pre>
- * topology.metrics.consumer.register:
- * - class: "org.apache.stormcrawler.opensearch.metrics.MetricsConsumer"
- * parallelism.hint: 1
- * argument: "yyyy-MM-dd"
- * </pre>
- */
-public class MetricsConsumer implements IMetricsConsumer {
-
- private final Logger LOG = LoggerFactory.getLogger(getClass());
-
- private static final String OSBoltType = "metrics";
-
- /** name of the index to use for the metrics (default : metrics) * */
- private static final String OSMetricsIndexNameParamName =
- "opensearch." + OSBoltType + ".index.name";
-
- private String indexName;
-
- private OpenSearchConnection connection;
-
- private String stormID;
-
- /** optional date format passed as argument, must be parsable as a SimpleDateFormat */
- private SimpleDateFormat dateFormat;
-
- @Override
- public void prepare(
- Map<String, Object> stormConf,
- Object registrationArgument,
- TopologyContext context,
- IErrorReporter errorReporter) {
- indexName = ConfUtils.getString(stormConf, OSMetricsIndexNameParamName, "metrics");
- stormID = context.getStormId();
- if (registrationArgument != null) {
- dateFormat = new SimpleDateFormat((String) registrationArgument, Locale.ROOT);
- LOG.info("Using date format {}", registrationArgument);
- }
- try {
- connection = OpenSearchConnection.getConnection(stormConf, OSBoltType);
- } catch (Exception e1) {
- LOG.error("Can't connect to OpenSearch", e1);
- throw new RuntimeException(e1);
- }
-
- // create a template if it doesn't exist
- try {
- IndexCreation.checkOrCreateIndexTemplate(connection.getClient(), OSBoltType, LOG);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void cleanup() {
- if (connection != null) {
- connection.close();
- }
- }
-
- @Override
- public void handleDataPoints(TaskInfo taskInfo, Collection<DataPoint> dataPoints) {
- final Date now = new Date();
- for (DataPoint dataPoint : dataPoints) {
- handleDataPoints(taskInfo, dataPoint.name, dataPoint.value, now);
- }
- }
-
- private void handleDataPoints(
- final TaskInfo taskInfo, final String nameprefix, final Object value, final Date now) {
- if (value instanceof Number) {
- indexDataPoint(taskInfo, now, nameprefix, ((Number) value).doubleValue());
- } else if (value instanceof Map) {
- for (Entry<String, Object> entry : ((Map<String, Object>) value).entrySet()) {
- String newnameprefix = nameprefix + "." + entry.getKey();
- handleDataPoints(taskInfo, newnameprefix, entry.getValue(), now);
- }
- } else if (value instanceof Collection) {
- for (Object collectionObj : (Collection<Object>) value) {
- handleDataPoints(taskInfo, nameprefix, collectionObj, now);
- }
- } else {
- LOG.warn("Found data point value {} of {}", nameprefix, value.getClass().toString());
- }
- }
-
- /**
- * Returns the name of the index that metrics will be written to.
- *
- * @return elastic index name
- */
- private String getIndexName(Date timestamp) {
- if (dateFormat == null) {
- return indexName;
- }
-
- StringBuilder sb = new StringBuilder(indexName);
- sb.append("-").append(dateFormat.format(timestamp));
- return sb.toString();
- }
-
- private void indexDataPoint(TaskInfo taskInfo, Date timestamp, String name, double value) {
- try {
- Map<String, Object> doc = new HashMap<>();
- doc.put("stormId", stormID);
- doc.put("srcComponentId", taskInfo.srcComponentId);
- doc.put("srcTaskId", taskInfo.srcTaskId);
- doc.put("srcWorkerHost", taskInfo.srcWorkerHost);
- doc.put("srcWorkerPort", taskInfo.srcWorkerPort);
- doc.put("name", name);
- doc.put("value", value);
- doc.put("timestamp", timestamp.toInstant().toString());
-
- final String targetIndex = getIndexName(timestamp);
- BulkOperation op =
- BulkOperation.of(b -> b.index(idx -> idx.index(targetIndex).document(doc)));
- connection.addToProcessor(op);
- } catch (Exception e) {
- LOG.error("problem when building request for OpenSearch", e);
- }
- }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java
deleted file mode 100644
index d3b654a0..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.metrics;
-
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.Histogram;
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.MetricFilter;
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.ScheduledReporter;
-import com.codahale.metrics.Timer;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.concurrent.TimeUnit;
-import org.apache.storm.metrics2.reporters.ScheduledStormReporter;
-import org.apache.stormcrawler.opensearch.IndexCreation;
-import org.apache.stormcrawler.opensearch.OpenSearchConnection;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.opensearch.core.bulk.BulkOperation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Storm V2 metrics reporter that writes metrics to an OpenSearch index with the same document
- * structure as the V1 {@link MetricsConsumer}. This allows existing OpenSearch dashboards to work
- * unchanged during migration from V1 to V2 metrics.
- *
- * <p>Configuration in storm.yaml:
- *
- * <pre>
- * storm.metrics.reporters:
- * - class: "org.apache.stormcrawler.opensearch.metrics.MetricsReporter"
- * report.period: 10
- * report.period.units: "SECONDS"
- * </pre>
- */
-public class MetricsReporter extends ScheduledStormReporter {
-
- private static final Logger LOG = LoggerFactory.getLogger(MetricsReporter.class);
-
- private static final String OSBoltType = "metrics";
-
- private static final String OSMetricsIndexNameParamName =
- "opensearch." + OSBoltType + ".index.name";
-
- private static final String DATE_FORMAT_KEY = "opensearch.metrics.date.format";
-
- private ScheduledReporter reporter;
-
- @Override
- public void prepare(
- MetricRegistry metricsRegistry,
- Map<String, Object> topoConf,
- Map<String, Object> reporterConf) {
-
- String indexName = ConfUtils.getString(topoConf, OSMetricsIndexNameParamName, "metrics");
- String stormId = (String) topoConf.getOrDefault("storm.id", "unknown");
-
- SimpleDateFormat dateFormat = null;
- String dateFormatStr = ConfUtils.getString(topoConf, DATE_FORMAT_KEY, null);
- if (dateFormatStr != null) {
- dateFormat = new SimpleDateFormat(dateFormatStr, Locale.ROOT);
- }
-
- OpenSearchConnection connection;
- try {
- connection = OpenSearchConnection.getConnection(topoConf, OSBoltType);
- } catch (Exception e) {
- LOG.error("Can't connect to OpenSearch", e);
- throw new RuntimeException(e);
- }
-
- try {
- IndexCreation.checkOrCreateIndexTemplate(connection.getClient(), OSBoltType, LOG);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- TimeUnit reportPeriodUnit = getReportPeriodUnit(reporterConf);
- long reportPeriod = getReportPeriod(reporterConf);
-
- reporter =
- new OpenSearchScheduledReporter(
- metricsRegistry, indexName, stormId, dateFormat, connection);
-
- reporter.start(reportPeriod, reportPeriodUnit);
- }
-
- @Override
- public void start() {
- // already started in prepare()
- }
-
- @Override
- public void stop() {
- if (reporter != null) {
- reporter.stop();
- }
- }
-
- /**
- * Inner ScheduledReporter that writes Codahale metrics to OpenSearch in the same format as the
- * V1 {@link MetricsConsumer}.
- */
- private static class OpenSearchScheduledReporter extends ScheduledReporter {
-
- private final String indexName;
- private final String stormId;
- private final SimpleDateFormat dateFormat;
- private final OpenSearchConnection connection;
-
- OpenSearchScheduledReporter(
- MetricRegistry registry,
- String indexName,
- String stormId,
- SimpleDateFormat dateFormat,
- OpenSearchConnection connection) {
- super(
- registry,
- "opensearch-metrics-reporter",
- MetricFilter.ALL,
- TimeUnit.SECONDS,
- TimeUnit.MILLISECONDS);
- this.indexName = indexName;
- this.stormId = stormId;
- this.dateFormat = dateFormat;
- this.connection = connection;
- }
-
- @Override
- @SuppressWarnings("rawtypes")
- public void report(
- SortedMap<String, Gauge> gauges,
- SortedMap<String, Counter> counters,
- SortedMap<String, Histogram> histograms,
- SortedMap<String, Meter> meters,
- SortedMap<String, Timer> timers) {
-
- Date now = new Date();
-
- for (Map.Entry<String, Gauge> entry : gauges.entrySet()) {
- Object value = entry.getValue().getValue();
- if (value instanceof Number) {
- indexDataPoint(now, entry.getKey(), ((Number) value).doubleValue());
- } else if (value instanceof Map) {
- for (Map.Entry<?, ?> mapEntry : ((Map<?, ?>) value).entrySet()) {
- if (mapEntry.getValue() instanceof Number) {
- indexDataPoint(
- now,
- entry.getKey() + "." + mapEntry.getKey(),
- ((Number) mapEntry.getValue()).doubleValue());
- }
- }
- }
- }
-
- for (Map.Entry<String, Counter> entry : counters.entrySet()) {
- indexDataPoint(now, entry.getKey(), entry.getValue().getCount());
- }
-
- for (Map.Entry<String, Histogram> entry : histograms.entrySet()) {
- indexDataPoint(now, entry.getKey(), entry.getValue().getSnapshot().getMean());
- }
-
- for (Map.Entry<String, Meter> entry : meters.entrySet()) {
- indexDataPoint(now, entry.getKey(), entry.getValue().getOneMinuteRate());
- }
-
- for (Map.Entry<String, Timer> entry : timers.entrySet()) {
- indexDataPoint(now, entry.getKey(), entry.getValue().getSnapshot().getMean());
- }
- }
-
- private String getIndexName(Date timestamp) {
- if (dateFormat == null) {
- return indexName;
- }
- return indexName + "-" + dateFormat.format(timestamp);
- }
-
- private void indexDataPoint(Date timestamp, String name, double value) {
- try {
- Map<String, Object> doc = new HashMap<>();
- doc.put("stormId", stormId);
- doc.put("name", name);
- doc.put("value", value);
- doc.put("timestamp", timestamp.toInstant().toString());
-
- final String targetIndex = getIndexName(timestamp);
- BulkOperation op =
- BulkOperation.of(b -> b.index(idx -> idx.index(targetIndex).document(doc)));
- connection.addToProcessor(op);
- } catch (Exception e) {
- LOG.error("Problem when building request for OpenSearch", e);
- }
- }
- }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
deleted file mode 100644
index c74184c2..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.metrics;
-
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.storm.Config;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.topology.OutputFieldsDeclarer;
-import org.apache.storm.topology.base.BaseRichBolt;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.utils.TupleUtils;
-import org.apache.stormcrawler.metrics.CrawlerMetrics;
-import org.apache.stormcrawler.opensearch.Constants;
-import org.apache.stormcrawler.opensearch.OpenSearchConnection;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.opensearch.OpenSearchClient;
-import org.opensearch.client.opensearch._types.FieldValue;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Queries the status index periodically to get the count of URLs per status. This bolt can be
- * connected to the output of any other bolt and will not produce anything as output.
- */
-public class StatusMetricsBolt extends BaseRichBolt {
-
- private static final Logger LOG = LoggerFactory.getLogger(StatusMetricsBolt.class);
-
- private static final String OSBoltType = "status";
- private static final String OSStatusIndexNameParamName =
- Constants.PARAMPREFIX + "status.index.name";
-
- private String indexName;
-
- private OpenSearchClient client;
-
- private Map<String, Long> latestStatusCounts = new ConcurrentHashMap<>(6);
-
- private int freqStats = 60;
-
- private OutputCollector _collector;
-
- private transient StatusCounter[] counters;
-
- private static final class StatusCounter {
- final String name;
- final AtomicBoolean ready = new AtomicBoolean(true);
-
- StatusCounter(String name) {
- this.name = name;
- }
- }
-
- @Override
- public void prepare(
- Map<String, Object> stormConf, TopologyContext context, OutputCollector collector) {
- _collector = collector;
- indexName = ConfUtils.getString(stormConf, OSStatusIndexNameParamName, "status");
- try {
- client = OpenSearchConnection.getClient(stormConf, OSBoltType);
- } catch (Exception e1) {
- LOG.error("Can't connect to OpenSearch", e1);
- throw new RuntimeException(e1);
- }
-
- CrawlerMetrics.registerGauge(
- context, stormConf, "status.count", () -> latestStatusCounts, freqStats);
-
- counters = new StatusCounter[6];
-
- counters[0] = new StatusCounter("DISCOVERED");
- counters[1] = new StatusCounter("FETCHED");
- counters[2] = new StatusCounter("FETCH_ERROR");
- counters[3] = new StatusCounter("REDIRECTION");
- counters[4] = new StatusCounter("ERROR");
- counters[5] = new StatusCounter("TOTAL");
- }
-
- @Override
- public Map<String, Object> getComponentConfiguration() {
- Config conf = new Config();
- conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, freqStats);
- return conf;
- }
-
- @Override
- public void execute(Tuple input) {
- _collector.ack(input);
-
- // this bolt can be connected to anything
- // we just want to trigger a new search when the input is a tick tuple
- if (!TupleUtils.isTick(input)) {
- return;
- }
-
- for (StatusCounter counter : counters) {
- // still waiting for results from previous request
- if (!counter.ready.compareAndSet(true, false)) {
- LOG.debug("Not ready to get counts for status {}", counter.name);
- continue;
- }
- final String statusName = counter.name;
- CompletableFuture.supplyAsync(
- () -> {
- try {
- if (statusName.equalsIgnoreCase("TOTAL")) {
- return client.count(c -> c.index(indexName));
- } else {
- return client.count(
- c ->
- c.index(indexName)
- .query(
- q ->
- q.term(
- t ->
- t.field(
- "status")
- .value(
- FieldValue
- .of(
- statusName)))));
- }
- } catch (Exception e) {
- throw new CompletionException(e);
- }
- })
- .thenAccept(
- response -> {
- counter.ready.set(true);
- LOG.debug(
- "Got {} counts for status:{}",
- response.count(),
- statusName);
- latestStatusCounts.put(statusName, response.count());
- })
- .exceptionally(
- e -> {
- counter.ready.set(true);
- Throwable cause =
- e instanceof CompletionException ? e.getCause() : e;
- LOG.error(
- "Failure when getting counts for status:{}",
- statusName,
- cause);
- return null;
- });
- }
- }
-
- @Override
- public void cleanup() {
- if (client != null) {
- try {
- client._transport().close();
- } catch (Exception e) {
- LOG.error("Exception closing client transport", e);
- }
- }
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- // NONE - THIS BOLT DOES NOT GET CONNECTED TO ANY OTHERS
- }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
deleted file mode 100644
index 6f6d7be6..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.parse.filter;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import java.util.Map;
-import org.apache.stormcrawler.opensearch.DelegateRefresher;
-import org.apache.stormcrawler.parse.ParseFilter;
-import org.apache.stormcrawler.parse.ParseResult;
-import org.jetbrains.annotations.NotNull;
-import org.w3c.dom.DocumentFragment;
-
-/**
- * Wraps a ParseFilter whose resources are in a JSON file that can be stored in OpenSearch. The
- * benefit of doing this is that the resources can be refreshed automatically and modified without
- * having to recompile the jar and restart the topology. The connection to OpenSearch is done via
- * the config and uses a new bolt type 'config'.
- *
- * <p>The configuration of the delegate is done in the parsefilters.json as usual.
- *
- * <pre>
- * {
- * "class": "org.apache.stormcrawler.opensearch.parse.filter.JSONResourceWrapper",
- * "name": "OpenSearchCollectionTagger",
- * "params": {
- * "refresh": "60",
- * "delegate": {
- * "class": "org.apache.stormcrawler.parse.filter.CollectionTagger",
- * "params": {
- * "file": "collections.json"
- * }
- * }
- * }
- * }
- * </pre>
- *
- * The resource file can be pushed to OpenSearch with
- *
- * <pre>
- * curl -XPUT "$OSHOST/config/_create/collections.json" -H 'Content-Type: application/json' -d @src/main/resources/collections.json
- * </pre>
- */
-public class JSONResourceWrapper extends ParseFilter {
-
- private DelegateRefresher<ParseFilter> refresher;
-
- public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode filterParams) {
- refresher =
- new DelegateRefresher<>(
- ParseFilter.class, stormConf, filterParams, ParseFilter::configure);
- }
-
- @Override
- public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) {
- refresher.getDelegate().filter(url, content, doc, parse);
- }
-
- @Override
- public void cleanup() {
- if (refresher != null) {
- refresher.cleanup();
- }
- }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
deleted file mode 100644
index 0afe2b19..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
+++ /dev/null
@@ -1,330 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.persistence;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import org.apache.storm.spout.SpoutOutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.opensearch.Constants;
-import org.apache.stormcrawler.opensearch.IndexCreation;
-import org.apache.stormcrawler.opensearch.OpenSearchConnection;
-import org.apache.stormcrawler.persistence.AbstractQueryingSpout;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.json.JsonData;
-import org.opensearch.client.opensearch.OpenSearchClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class AbstractSpout extends AbstractQueryingSpout {
-
- private static final Logger LOG = LoggerFactory.getLogger(AbstractSpout.class);
-
- protected static final String OSBoltType = "status";
- protected static final String OSStatusIndexNameParamName =
- Constants.PARAMPREFIX + OSBoltType + ".index.name";
-
- /** Field name to use for aggregating * */
- protected static final String OSStatusBucketFieldParamName =
- Constants.PARAMPREFIX + OSBoltType + ".bucket.field";
-
- protected static final String OSStatusMaxBucketParamName =
- Constants.PARAMPREFIX + OSBoltType + ".max.buckets";
- protected static final String OSStatusMaxURLsParamName =
- Constants.PARAMPREFIX + OSBoltType + ".max.urls.per.bucket";
-
- /** Field name to use for sorting the URLs within a bucket, not used if empty or null. */
- protected static final String OSStatusBucketSortFieldParamName =
- Constants.PARAMPREFIX + OSBoltType + ".bucket.sort.field";
-
- /** Field name to use for sorting the buckets, not used if empty or null. */
- protected static final String OSStatusGlobalSortFieldParamName =
- Constants.PARAMPREFIX + OSBoltType + ".global.sort.field";
-
- protected static final String OSStatusFilterParamName =
- Constants.PARAMPREFIX + OSBoltType + ".filterQuery";
-
- protected static final String OSStatusQueryTimeoutParamName =
- Constants.PARAMPREFIX + OSBoltType + ".query.timeout";
-
- /** Query to use as a positive filter, set by es.status.filterQuery */
- protected List<String> filterQueries = null;
-
- protected String indexName;
-
- protected static OpenSearchClient client;
-
- /**
- * when using multiple instances - each one is in charge of a specific shard useful when
- * sharding based on host or domain to guarantee a good mix of URLs
- */
- protected int shardID = -1;
-
- /** Used to distinguish between instances in the logs * */
- protected String logIdprefix = "";
-
- /** Field name used for field collapsing e.g. key * */
- protected String partitionField;
-
- protected int maxURLsPerBucket = 10;
-
- protected int maxBucketNum = 10;
-
- protected List<String> bucketSortField = new ArrayList<>();
-
- protected String totalSortField = "";
-
- protected Date queryDate;
-
- protected int queryTimeout = -1;
-
- /**
- * Single-threaded executor dedicated to running the asynchronous OpenSearch queries triggered
- * from this spout. Using a dedicated, named, daemon executor avoids polluting the JVM-wide
- * {@link java.util.concurrent.ForkJoinPool#commonPool() common pool} (which is shared with all
- * other Storm tasks running in the same worker) and allows clean shutdown when the topology is
- * stopped.
- */
- protected ExecutorService queryExecutor;
-
- @Override
- public void open(
- Map<String, Object> stormConf,
- TopologyContext context,
- SpoutOutputCollector collector) {
-
- super.open(stormConf, context, collector);
-
- final String executorName =
- "OpenSearchSpout-"
- + context.getThisComponentId()
- + "-"
- + context.getThisTaskIndex();
- queryExecutor =
- Executors.newSingleThreadExecutor(
- r -> {
- Thread t = new Thread(r, executorName);
- t.setDaemon(true);
- return t;
- });
-
- indexName = ConfUtils.getString(stormConf, OSStatusIndexNameParamName, "status");
-
- // one OS client per JVM
- synchronized (AbstractSpout.class) {
- try {
- if (client == null) {
- client = OpenSearchConnection.getClient(stormConf, OSBoltType);
- }
- } catch (Exception e1) {
- LOG.error("Can't connect to OpenSearch", e1);
- throw new RuntimeException(e1);
- }
-
- // use the default status schema if none has been specified
- try {
- IndexCreation.checkOrCreateIndex(client, indexName, OSBoltType, LOG);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- // if more than one instance is used we expect their number to be the
- // same as the number of shards
- int totalTasks = context.getComponentTasks(context.getThisComponentId()).size();
- if (totalTasks > 1) {
- logIdprefix =
- "[" + context.getThisComponentId() + " #" + context.getThisTaskIndex() + "] ";
-
- // determine the number of shards so that we can restrict the
- // search
-
- // TODO use the admin API when it gets available
- // TODO or the low level one with
- // https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-shards-stores.html
- // TODO identify local shards and use those if possible
-
- // ClusterSearchShardsRequest request = new
- // ClusterSearchShardsRequest(
- // indexName);
- // ClusterSearchShardsResponse shardresponse = client.admin()
- // .cluster().searchShards(request).actionGet();
- // ClusterSearchShardsGroup[] shardgroups =
- // shardresponse.getGroups();
- // if (totalTasks != shardgroups.length) {
- // throw new RuntimeException(
- // "Number of OS spout instances should be the same as number of
- // shards ("
- // + shardgroups.length + ") but is " + totalTasks);
- // }
- // shardID = shardgroups[context.getThisTaskIndex()].getShardId()
- // .getId();
-
- // TEMPORARY simply use the task index as shard index
- shardID = context.getThisTaskIndex();
- LOG.info("{} assigned shard ID {}", logIdprefix, shardID);
- }
-
- partitionField = ConfUtils.getString(stormConf, OSStatusBucketFieldParamName, "key");
-
- bucketSortField = ConfUtils.loadListFromConf(OSStatusBucketSortFieldParamName, stormConf);
-
- totalSortField = ConfUtils.getString(stormConf, OSStatusGlobalSortFieldParamName);
-
- maxURLsPerBucket = ConfUtils.getInt(stormConf, OSStatusMaxURLsParamName, 1);
- maxBucketNum = ConfUtils.getInt(stormConf, OSStatusMaxBucketParamName, 10);
-
- queryTimeout = ConfUtils.getInt(stormConf, OSStatusQueryTimeoutParamName, -1);
-
- filterQueries = ConfUtils.loadListFromConf(OSStatusFilterParamName, stormConf);
- }
-
- /** Builds a query and use it retrieve the results from OS * */
- protected abstract void populateBuffer();
-
- /**
- * Converts the {@link JsonData} carried by a search hit into a plain {@code Map<String,
- * Object>} populated with native Java types (strings, numbers, lists, nested maps).
- *
- * <p>This helper exists to work around a subtle short-circuit inside {@code JsonDataImpl#to}:
- *
- * <pre>{@code
- * if (clazz.isAssignableFrom(value.getClass())) {
- * return (T) value;
- * }
- * }</pre>
- *
- * <p>When {@link JsonData} is produced via {@code JsonData._DESERIALIZER} (as opposed to a raw
- * Jackson {@code TreeNode}), its internal {@code value} is a Jakarta JSON-P {@code JsonObject}
- * — typically a Parsson {@code JsonObjectImpl}. Crucially, {@code jakarta.json.JsonObject
- * extends Map<String, JsonValue>}, so calling {@code source.to(Map.class)} or {@code
- * source.to(Object.class)} matches the {@code isAssignableFrom} guard and returns the raw
- * {@code JsonObject} <em>without ever consulting the configured {@link
- * org.opensearch.client.json.jackson.JacksonJsonpMapper}</em>. The resulting map values are
- * still {@code JsonString}/{@code JsonStringImpl} instances and any downstream {@code (String)
- * value} cast blows up with a {@link ClassCastException}.
- *
- * <p>Passing a concrete subclass that {@code JsonObject} does <em>not</em> implement (here
- * {@link LinkedHashMap}) bypasses the short-circuit and forces the round-trip through {@code
- * JacksonJsonpGenerator -> JacksonJsonpParser -> Jackson ObjectMapper.readValue}, which yields
- * native Java values (String, Long, Double, ArrayList, LinkedHashMap…).
- *
- * @param source the document source carried by an OpenSearch hit; may be {@code null}
- * @return a (possibly empty) map of native Java values, never {@code null}
- */
- static Map<String, Object> sourceAsMap(JsonData source) {
- if (source == null) {
- return Collections.emptyMap();
- }
- @SuppressWarnings("unchecked")
- Map<String, Object> map = source.to(LinkedHashMap.class);
- return map == null ? Collections.emptyMap() : map;
- }
-
- /**
- * Adds a document source to the URL buffer unless it is already being processed.
- *
- * @param source the document source as a key-value map (must contain a "url" entry)
- * @return {@code true} if the URL was added to the buffer, {@code false} if it was already
- * being processed or already present
- */
- protected final boolean addHitToBuffer(Map<String, Object> source) {
- String url = (String) source.get("url");
- // is already being processed - skip it!
- if (beingProcessed.containsKey(url)) {
- return false;
- }
- return buffer.add(url, fromKeyValues(source));
- }
-
- protected final Metadata fromKeyValues(Map<String, Object> keyValues) {
- Map<String, List<String>> mdAsMap = (Map<String, List<String>>) keyValues.get("metadata");
- Metadata metadata = new Metadata();
- if (mdAsMap != null) {
- for (Entry<String, List<String>> mdEntry : mdAsMap.entrySet()) {
- String key = mdEntry.getKey();
- // periods are not allowed - replace with %2E
- key = key.replaceAll("%2E", "\\.");
- Object mdValObj = mdEntry.getValue();
- // single value
- if (mdValObj instanceof String) {
- metadata.addValue(key, (String) mdValObj);
- } else {
- // multi valued
- metadata.addValues(key, (List<String>) mdValObj);
- }
- }
- }
- return metadata;
- }
-
- @Override
- public void ack(Object msgId) {
- LOG.debug("{} Ack for {}", logIdprefix, msgId);
- super.ack(msgId);
- }
-
- @Override
- public void fail(Object msgId) {
- LOG.info("{} Fail for {}", logIdprefix, msgId);
- super.fail(msgId);
- }
-
- @Override
- public void close() {
- // Shut the dedicated query executor down first so no more requests can be enqueued
- // against the client while we are tearing it down. We give in-flight queries a short
- // grace period to drain to keep shutdown reasonably graceful.
- if (queryExecutor != null) {
- queryExecutor.shutdown();
- try {
- if (!queryExecutor.awaitTermination(30, TimeUnit.SECONDS)) {
- LOG.warn(
- "{} Query executor did not terminate within 30s, forcing shutdown",
- logIdprefix);
- queryExecutor.shutdownNow();
- }
- } catch (InterruptedException e) {
- queryExecutor.shutdownNow();
- Thread.currentThread().interrupt();
- }
- queryExecutor = null;
- }
-
- synchronized (AbstractSpout.class) {
- if (client != null) {
- try {
- client._transport().close();
- } catch (IOException e) {
- LOG.error("Exception caught when closing client", e);
- }
- client = null;
- }
- }
- }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java
deleted file mode 100644
index a6272f36..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java
+++ /dev/null
@@ -1,432 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.persistence;
-
-import java.io.IOException;
-import java.time.Instant;
-import java.time.ZoneOffset;
-import java.time.format.DateTimeFormatter;
-import java.util.Calendar;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.storm.spout.SpoutOutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.opensearch.Constants;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.json.JsonData;
-import org.opensearch.client.opensearch._types.FieldValue;
-import org.opensearch.client.opensearch._types.SortOrder;
-import org.opensearch.client.opensearch._types.aggregations.Aggregate;
-import org.opensearch.client.opensearch._types.aggregations.Aggregation;
-import org.opensearch.client.opensearch._types.aggregations.StringTermsBucket;
-import org.opensearch.client.opensearch._types.aggregations.TopHitsAggregate;
-import org.opensearch.client.opensearch.core.SearchRequest;
-import org.opensearch.client.opensearch.core.SearchResponse;
-import org.opensearch.client.opensearch.core.search.Hit;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Spout which pulls URL from an OpenSearch index. Use a single instance unless you use
- * 'opensearch.status.routing' with the StatusUpdaterBolt, in which case you need to have exactly
- * the same number of spout instances as OpenSearch shards. Guarantees a good mix of URLs by
- * aggregating them by an arbitrary field e.g. key.
- */
-public class AggregationSpout extends AbstractSpout {
-
- private static final Logger LOG = LoggerFactory.getLogger(AggregationSpout.class);
-
- private static final String StatusSampleParamName = Constants.PARAMPREFIX + "status.sample";
- private static final String MostRecentDateIncreaseParamName =
- Constants.PARAMPREFIX + "status.recentDate.increase";
- private static final String MostRecentDateMinGapParamName =
- Constants.PARAMPREFIX + "status.recentDate.min.gap";
-
- private boolean sample = false;
-
- private int recentDateIncrease = -1;
- private int recentDateMinGap = -1;
-
- protected Set<String> currentBuckets;
-
- @Override
- public void open(
- Map<String, Object> stormConf,
- TopologyContext context,
- SpoutOutputCollector collector) {
- sample = ConfUtils.getBoolean(stormConf, StatusSampleParamName, sample);
- recentDateIncrease =
- ConfUtils.getInt(stormConf, MostRecentDateIncreaseParamName, recentDateIncrease);
- recentDateMinGap =
- ConfUtils.getInt(stormConf, MostRecentDateMinGapParamName, recentDateMinGap);
- super.open(stormConf, context, collector);
- currentBuckets = new HashSet<>();
- }
-
- @Override
- protected void populateBuffer() {
-
- if (queryDate == null) {
- queryDate = new Date();
- lastTimeResetToNow = Instant.now();
- }
-
- String formattedQueryDate =
- Instant.ofEpochMilli(queryDate.getTime())
- .atOffset(ZoneOffset.UTC)
- .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
-
- LOG.info("{} Populating buffer with nextFetchDate <= {}", logIdprefix, formattedQueryDate);
-
- // Build the top_hits sub-aggregation
- Aggregation topHitsAgg =
- Aggregation.of(
- a ->
- a.topHits(
- th -> {
- th.size(maxURLsPerBucket).explain(false);
- for (String bsf : bucketSortField) {
- th.sort(
- s ->
- s.field(
- fs ->
- fs.field(bsf)
- .order(
- SortOrder
- .Asc)));
- }
- return th;
- }));
-
- // Build the terms (partition) aggregation with top_hits sub-agg
- Aggregation.Builder.ContainerBuilder partitionAggBuilder =
- new Aggregation.Builder()
- .terms(
- t -> {
- t.field(partitionField).size(maxBucketNum);
- // sort between buckets by the min sub-aggregation
- if (StringUtils.isNotBlank(totalSortField)) {
- t.order(
- Collections.singletonList(
- Collections.singletonMap(
- "top_hit", SortOrder.Asc)));
- }
- return t;
- })
- .aggregations("docs", topHitsAgg);
-
- // add the min sub-aggregation used for sorting between buckets
- if (StringUtils.isNotBlank(totalSortField)) {
- partitionAggBuilder.aggregations(
- "top_hit", Aggregation.of(minAgg -> minAgg.min(m -> m.field(totalSortField))));
- }
-
- Aggregation partitionAgg = partitionAggBuilder.build();
-
- // Build the search request
- SearchRequest.Builder requestBuilder =
- new SearchRequest.Builder()
- .index(indexName)
- .size(0)
- .trackTotalHits(t -> t.enabled(false))
- .query(
- q ->
- q.bool(
- b -> {
- b.filter(
- f ->
- f.range(
- r ->
- r.field(
- "nextFetchDate")
- .lte(
- JsonData
- .of(
- formattedQueryDate))));
- if (filterQueries != null) {
- for (String fq : filterQueries) {
- b.filter(
- f ->
- f.queryString(
- qs ->
- qs
- .query(
- fq)));
- }
- }
- return b;
- }));
-
- if (queryTimeout != -1) {
- requestBuilder.timeout(queryTimeout + "s");
- }
-
- if (sample) {
- // Wrap in a diversified sampler aggregation
- requestBuilder.aggregations(
- "sample",
- Aggregation.of(
- a ->
- a.diversifiedSampler(
- ds ->
- ds.field(partitionField)
- .maxDocsPerValue(
- maxURLsPerBucket)
- .shardSize(
- maxURLsPerBucket
- * maxBucketNum))
- .aggregations("partition", partitionAgg)));
- } else {
- requestBuilder.aggregations("partition", partitionAgg);
- }
-
- // shard preference for routing
- if (shardID != -1) {
- requestBuilder.preference("_shards:" + shardID + "|_local");
- }
-
- SearchRequest request = requestBuilder.build();
-
- // dump query to log
- LOG.debug("{} OpenSearch query {}", logIdprefix, request);
-
- LOG.trace("{} isInQuery set to true", logIdprefix);
- isInQuery.set(true);
-
- CompletableFuture.supplyAsync(
- () -> {
- try {
- return client.search(request, JsonData.class);
- } catch (IOException e) {
- throw new CompletionException(e);
- }
- },
- queryExecutor)
- .thenAccept(this::handleResponse)
- .exceptionally(
- e -> {
- Throwable cause = e instanceof CompletionException ? e.getCause() : e;
- LOG.error("{} Exception with OpenSearch query", logIdprefix, cause);
- markQueryReceivedNow();
- return null;
- });
- }
-
- /**
- * Handles the search response from an asynchronous aggregation query, extracting URLs from term
- * buckets and adding them to the buffer.
- *
- * @param response the search response containing aggregation results
- */
- protected void handleResponse(SearchResponse<JsonData> response) {
- long timeTaken = System.currentTimeMillis() - getTimeLastQuerySent();
-
- Map<String, Aggregate> aggregs = response.aggregations();
-
- if (aggregs == null || aggregs.isEmpty()) {
- markQueryReceivedNow();
- return;
- }
-
- // Unwrap the sample aggregation if present
- Aggregate sampleAgg = aggregs.get("sample");
- if (sampleAgg != null) {
- aggregs = sampleAgg.sampler().aggregations();
- }
-
- Aggregate partitionAgg = aggregs.get("partition");
- List<StringTermsBucket> buckets = partitionAgg.sterms().buckets().array();
-
- int numhits = 0;
- int numBuckets = 0;
- int alreadyprocessed = 0;
-
- Instant mostRecentDateFound = null;
-
- currentBuckets.clear();
-
- // For each entry
- Iterator<StringTermsBucket> iterator = buckets.iterator();
- while (iterator.hasNext()) {
- StringTermsBucket entry = iterator.next();
- String key = entry.key(); // bucket key
-
- currentBuckets.add(key);
-
- long docCount = entry.docCount(); // Doc count
-
- int hitsForThisBucket = 0;
-
- List<FieldValue> lastSortValues = null;
- // filter results so that we don't include URLs we are already
- // being processed
- TopHitsAggregate topHits = entry.aggregations().get("docs").topHits();
- for (Hit<JsonData> hit : topHits.hits().hits()) {
-
- Map<String, Object> keyValues = sourceAsMap(hit.source());
-
- LOG.debug("{} -> id [{}], _source [{}]", logIdprefix, hit.id(), keyValues);
-
- hitsForThisBucket++;
-
- lastSortValues = hit.sort();
-
- String url = (String) keyValues.get("url");
-
- // consider only the first document of the last bucket
- // for optimising the nextFetchDate
- if (hitsForThisBucket == 1 && !iterator.hasNext()) {
- String strDate = (String) keyValues.get("nextFetchDate");
- try {
- mostRecentDateFound = Instant.parse(strDate);
- } catch (Exception e) {
- throw new RuntimeException("can't parse date :" + strDate);
- }
- }
-
- // is already being processed or in buffer - skip it!
- if (beingProcessed.containsKey(url)) {
- LOG.debug("{} -> already processed: {}", logIdprefix, url);
- alreadyprocessed++;
- continue;
- }
-
- Metadata metadata = fromKeyValues(keyValues);
- boolean added = buffer.add(url, metadata);
- if (!added) {
- LOG.debug("{} -> already in buffer: {}", logIdprefix, url);
- alreadyprocessed++;
- continue;
- }
- LOG.debug("{} -> added to buffer : {}", logIdprefix, url);
- }
-
- if (lastSortValues != null && !lastSortValues.isEmpty()) {
- sortValuesForKey(key, lastSortValues.toArray());
- }
-
- if (hitsForThisBucket > 0) {
- numBuckets++;
- }
-
- numhits += hitsForThisBucket;
-
- LOG.debug(
- "{} key [{}], hits[{}], doc_count [{}], already_processed [{}]",
- logIdprefix,
- key,
- hitsForThisBucket,
- docCount,
- alreadyprocessed);
- }
-
- LOG.info(
- "{} OpenSearch query returned {} hits from {} buckets in {} msec with {} already being processed. Took {} msec per doc on average.",
- logIdprefix,
- numhits,
- numBuckets,
- timeTaken,
- alreadyprocessed,
- ((float) timeTaken / numhits));
-
- queryTimes.accept(timeTaken);
- eventCounter.scope("already_being_processed").incrBy(alreadyprocessed);
- eventCounter.scope("OpenSearch_queries").incrBy(1);
- eventCounter.scope("OpenSearch_docs").incrBy(numhits);
-
- // optimise the nextFetchDate by getting the most recent value
- // returned in the query and add to it, unless the previous value is
- // within n mins in which case we'll keep it
- if (mostRecentDateFound != null && recentDateIncrease >= 0) {
- Calendar potentialNewDate =
- Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
- potentialNewDate.setTimeInMillis(mostRecentDateFound.toEpochMilli());
- potentialNewDate.add(Calendar.MINUTE, recentDateIncrease);
- Date oldDate = null;
- // check boundaries
- if (this.recentDateMinGap > 0) {
- Calendar low = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
- low.setTime(queryDate);
- low.add(Calendar.MINUTE, -recentDateMinGap);
- Calendar high = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
- high.setTime(queryDate);
- high.add(Calendar.MINUTE, recentDateMinGap);
- if (high.before(potentialNewDate) || low.after(potentialNewDate)) {
- oldDate = queryDate;
- }
- } else {
- oldDate = queryDate;
- }
- if (oldDate != null) {
- queryDate = potentialNewDate.getTime();
- LOG.info(
- "{} queryDate changed from {} to {} based on mostRecentDateFound {}",
- logIdprefix,
- oldDate,
- queryDate,
- mostRecentDateFound);
- } else {
- LOG.info(
- "{} queryDate kept at {} based on mostRecentDateFound {}",
- logIdprefix,
- queryDate,
- mostRecentDateFound);
- }
- }
-
- // reset the value for next fetch date if the previous one is too old
- if (resetFetchDateAfterNSecs != -1) {
- Instant changeNeededOn =
- Instant.ofEpochMilli(
- lastTimeResetToNow.toEpochMilli() + (resetFetchDateAfterNSecs * 1000L));
- if (Instant.now().isAfter(changeNeededOn)) {
- LOG.info(
- "{} queryDate set to null based on resetFetchDateAfterNSecs {}",
- logIdprefix,
- resetFetchDateAfterNSecs);
- queryDate = null;
- }
- }
-
- // change the date if we don't get any results at all
- if (numBuckets == 0) {
- queryDate = null;
- }
-
- // remove lock
- markQueryReceivedNow();
- }
-
- /**
- * Receives the sort values of the last hit returned for a bucket, as the {@link FieldValue}
- * instances of the client. Subclasses can pass them to {@code search_after} to read the bucket
- * from that position.
- */
- protected void sortValuesForKey(String key, Object[] sortValues) {}
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
deleted file mode 100644
index b3301e5c..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.persistence;
-
-import com.github.benmanes.caffeine.cache.Cache;
-import com.github.benmanes.caffeine.cache.Caffeine;
-import java.io.IOException;
-import java.time.Instant;
-import java.time.ZoneOffset;
-import java.time.format.DateTimeFormatter;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
-import org.apache.storm.spout.SpoutOutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.stormcrawler.opensearch.Constants;
-import org.apache.stormcrawler.persistence.EmptyQueueListener;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.json.JsonData;
-import org.opensearch.client.opensearch._types.FieldValue;
-import org.opensearch.client.opensearch._types.SortOrder;
-import org.opensearch.client.opensearch.core.SearchRequest;
-import org.opensearch.client.opensearch.core.SearchResponse;
-import org.opensearch.client.opensearch.core.search.Hit;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Uses collapsing spouts to get an initial set of URLs and keys to query for and gets emptyQueue
- * notifications from the URLBuffer to query OpenSearch for a specific key.
- *
- * @since 1.15
- */
-public class HybridSpout extends AggregationSpout implements EmptyQueueListener {
-
- private static final Logger LOG = LoggerFactory.getLogger(HybridSpout.class);
-
- protected static final String RELOADPARAMNAME =
- Constants.PARAMPREFIX + "status.max.urls.per.reload";
-
- private int bufferReloadSize = 10;
-
- private Cache<String, Object[]> searchAfterCache;
-
- private HostResultListener hrl;
-
- @Override
- public void open(
- Map<String, Object> stormConf,
- TopologyContext context,
- SpoutOutputCollector collector) {
- super.open(stormConf, context, collector);
- bufferReloadSize = ConfUtils.getInt(stormConf, RELOADPARAMNAME, maxURLsPerBucket);
- buffer.setEmptyQueueListener(this);
- searchAfterCache = Caffeine.newBuilder().build();
- hrl = new HostResultListener();
- }
-
- @Override
- public void emptyQueue(String queueName) {
-
- LOG.info("{} Emptied buffer queue for {}", logIdprefix, queueName);
-
- if (!currentBuckets.contains(queueName)) {
- // not interested in this one any more
- return;
- }
-
- // reloading the aggregs - searching now
- // would just overload OpenSearch and yield
- // mainly duplicates
- if (isInQuery.get()) {
- LOG.trace("{} isInquery true for {}", logIdprefix, queueName);
- return;
- }
-
- LOG.info("{} Querying for more docs for {}", logIdprefix, queueName);
-
- if (queryDate == null) {
- queryDate = new Date();
- lastTimeResetToNow = Instant.now();
- }
-
- String formattedQueryDate =
- Instant.ofEpochMilli(queryDate.getTime())
- .atOffset(ZoneOffset.UTC)
- .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
-
- SearchRequest.Builder requestBuilder =
- new SearchRequest.Builder()
- .index(indexName)
- .size(bufferReloadSize)
- .trackTotalHits(t -> t.enabled(false))
- .query(
- q ->
- q.bool(
- b ->
- b.filter(
- f ->
- f.range(
- r ->
- r.field(
- "nextFetchDate")
- .lte(
- JsonData
- .of(
- formattedQueryDate))))
- .filter(
- f ->
- f.term(
- t ->
- t.field(
- partitionField)
- .value(
- FieldValue
- .of(
- queueName))))));
-
- // sort within a bucket
- for (String bsf : bucketSortField) {
- requestBuilder.sort(s -> s.field(fs -> fs.field(bsf).order(SortOrder.Asc)));
- }
-
- // do we have a search after for this one?
- Object[] searchAfterValues = searchAfterCache.getIfPresent(queueName);
- if (searchAfterValues != null) {
- for (Object sav : searchAfterValues) {
- requestBuilder.searchAfter((FieldValue) sav);
- }
- }
-
- // shard preference for routing
- if (shardID != -1) {
- requestBuilder.preference("_shards:" + shardID + "|_local");
- }
-
- SearchRequest request = requestBuilder.build();
-
- // dump query to log
- LOG.debug("{} OpenSearch query {} - {}", logIdprefix, queueName, request);
-
- CompletableFuture.supplyAsync(
- () -> {
- try {
- return client.search(request, JsonData.class);
- } catch (IOException e) {
- throw new CompletionException(e);
- }
- },
- queryExecutor)
- .thenAccept(hrl::handleResponse)
- .exceptionally(
- e -> {
- Throwable cause = e instanceof CompletionException ? e.getCause() : e;
- LOG.error("Exception with OpenSearch query", cause);
- return null;
- });
- }
-
- /** Overrides the handling of responses for aggregations. */
- @Override
- protected void handleResponse(SearchResponse<JsonData> response) {
- // delete all entries from the searchAfterCache when
- // we get the results from the aggregation spouts
- searchAfterCache.invalidateAll();
- super.handleResponse(response);
- }
-
- /** The aggregation kindly told us where to start from. */
- @Override
- protected void sortValuesForKey(String key, Object[] sortValues) {
- if (sortValues != null && sortValues.length > 0) {
- this.searchAfterCache.put(key, sortValues);
- }
- }
-
- /** Handling of results for a specific queue. */
- class HostResultListener {
-
- /**
- * Handles the search response for a host-specific query, extracting hits and adding them to
- * the buffer.
- *
- * @param response the search response containing document hits
- */
- void handleResponse(SearchResponse<JsonData> response) {
-
- int alreadyprocessed = 0;
- int numDocs = 0;
-
- List<Hit<JsonData>> hits = response.hits().hits();
-
- Object[] sortValues = null;
-
- // retrieve the key for these results
- String key = null;
-
- for (Hit<JsonData> hit : hits) {
- numDocs++;
-
- Map<String, Object> source = sourceAsMap(hit.source());
-
- String pfield = partitionField;
- Map<String, Object> fieldSource = source;
- if (pfield.startsWith("metadata.")) {
- @SuppressWarnings("unchecked")
- Map<String, Object> metadataMap = (Map<String, Object>) source.get("metadata");
- fieldSource = metadataMap;
- pfield = pfield.substring(9);
- }
- Object key_as_object = fieldSource.get(pfield);
- if (key_as_object instanceof List) {
- @SuppressWarnings("unchecked")
- List<String> keyList = (List<String>) key_as_object;
- if (keyList.size() == 1) {
- key = keyList.get(0);
- }
- } else {
- key = key_as_object.toString();
- }
-
- sortValues = hit.sort().toArray();
- if (!addHitToBuffer(source)) {
- alreadyprocessed++;
- }
- }
-
- // no key if no results have been found
- if (key != null) {
- searchAfterCache.put(key, sortValues);
- }
-
- eventCounter.scope("OpenSearch_queries_host").incrBy(1);
- eventCounter.scope("OpenSearch_docs_host").incrBy(numDocs);
- eventCounter.scope("already_being_processed_host").incrBy(alreadyprocessed);
-
- LOG.info(
- "{} OpenSearch term query returned {} hits in {} msec with {} already being processed for {}",
- logIdprefix,
- numDocs,
- response.took(),
- alreadyprocessed,
- key);
- }
- }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java
deleted file mode 100644
index 6af7d909..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.persistence;
-
-import com.github.benmanes.caffeine.cache.Cache;
-import com.github.benmanes.caffeine.cache.Caffeine;
-import java.io.IOException;
-import java.time.Instant;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.topology.OutputFieldsDeclarer;
-import org.apache.storm.topology.base.BaseRichBolt;
-import org.apache.storm.tuple.Tuple;
-import org.apache.stormcrawler.opensearch.AsyncBulkProcessor;
-import org.apache.stormcrawler.opensearch.Constants;
-import org.apache.stormcrawler.opensearch.IndexCreation;
-import org.apache.stormcrawler.opensearch.OpenSearchConnection;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.opensearch.core.BulkRequest;
-import org.opensearch.client.opensearch.core.BulkResponse;
-import org.opensearch.client.opensearch.core.bulk.BulkOperation;
-import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Sends information about the queues into an OpenSearch index. This has to be connected to a status
- * updater bolt.
- */
-public class QueueBolt extends BaseRichBolt implements AsyncBulkProcessor.Listener {
-
- private static final Logger LOG = LoggerFactory.getLogger(QueueBolt.class);
-
- private static final String OSBoltType = "queues";
-
- static final String OSQueuesIndexNameParamName =
- Constants.PARAMPREFIX + OSBoltType + ".index.name";
-
- /**
- * Parameter name to configure the cache of known queues. @see
- * https://github.com/ben-manes/caffeine/wiki/Specification Default value is
- * "maximumSize=10000".
- */
- static final String OSQueuesCacheSpecParamName =
- Constants.PARAMPREFIX + OSBoltType + ".cache.spec";
-
- private OutputCollector _collector;
-
- private String indexName;
-
- private OpenSearchConnection connection;
-
- /** Keys confirmed to be present in the index; populated on bulk success only */
- private Cache<String, Boolean> knownQueue;
-
- /** docID to key for the requests currently in the bulk processor */
- private final Map<String, String> inFlight = new ConcurrentHashMap<>();
-
- public QueueBolt() {}
-
- /** Sets the index name instead of taking it from the configuration. * */
- public QueueBolt(String indexName) {
- this.indexName = indexName;
- }
-
- @Override
- public void prepare(
- Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
- _collector = collector;
- if (indexName == null) {
- indexName = ConfUtils.getString(conf, QueueBolt.OSQueuesIndexNameParamName, OSBoltType);
- }
- try {
- connection = OpenSearchConnection.getConnection(conf, OSBoltType, this);
- } catch (Exception e1) {
- LOG.error("Can't connect to OpenSearch", e1);
- throw new RuntimeException(e1);
- }
- try {
- IndexCreation.checkOrCreateIndex(connection.getClient(), indexName, OSBoltType, LOG);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- final String cacheSpec =
- ConfUtils.getString(
- conf, QueueBolt.OSQueuesCacheSpecParamName, "maximumSize=10000");
- knownQueue = Caffeine.from(cacheSpec).build();
- }
-
- @Override
- public void cleanup() {
- if (connection != null) {
- connection.close();
- }
- }
-
- @Override
- public void execute(Tuple tuple) {
-
- final String key = tuple.getStringByField("key");
-
- // ack no matter what - the queue info is best effort bookkeeping
- // and failed writes are retried when the key is next seen
- _collector.ack(tuple);
-
- // check whether this key is already known
- if (knownQueue.getIfPresent(key) != null) {
- return;
- }
-
- final String docID = org.apache.commons.codec.digest.DigestUtils.sha256Hex(key);
-
- // a request for this key is already in the bulk processor
- if (inFlight.putIfAbsent(docID, key) != null) {
- return;
- }
-
- final HashMap<String, String> fields = new HashMap<>();
- fields.put("key", key);
- fields.put("lastUpdated", Instant.now().toString());
-
- final BulkOperation op =
- BulkOperation.of(b -> b.create(c -> c.index(indexName).id(docID).document(fields)));
-
- connection.addToProcessor(op);
- }
-
- @Override
- public void beforeBulk(long executionId, BulkRequest request) {
- LOG.debug("beforeBulk {} with {} actions", executionId, request.operations().size());
- }
-
- @Override
- public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
- LOG.debug("afterBulk [{}] with {} responses", executionId, response.items().size());
- for (BulkResponseItem item : response.items()) {
- final String key = inFlight.remove(item.id());
- if (key == null) {
- continue;
- }
- if (item.error() == null) {
- knownQueue.put(key, Boolean.TRUE);
- } else if (item.status() == 409) {
- // entry already exists in the index, e.g. written before a worker
- // restart or evicted from the cache - not an error
- knownQueue.put(key, Boolean.TRUE);
- } else {
- // do not cache the key so that the write is retried
- // when the key is next seen
- final var error = item.error();
- LOG.error(
- "Failed to index queue entry {}: {}",
- key,
- error.reason() != null ? error.reason() : error.type());
- }
- }
- }
-
- @Override
- public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
- LOG.error("Exception with bulk {} - failing the whole lot ", executionId, failure);
- for (BulkOperation op : request.operations()) {
- // not cached - the writes are retried when the keys are next seen
- inFlight.remove(OpenSearchConnection.getBulkOperationId(op));
- }
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- // nothing to do here - this bolt is the last of a topology
- }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
deleted file mode 100644
index 2dcd76fe..00000000
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
+++ /dev/null
@@ -1,358 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.persistence;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.tuple.Values;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.metrics.CrawlerMetrics;
-import org.apache.stormcrawler.metrics.ScopedCounter;
-import org.apache.stormcrawler.metrics.ScopedReducedMetric;
-import org.apache.stormcrawler.opensearch.AsyncBulkProcessor;
-import org.apache.stormcrawler.opensearch.Constants;
-import org.apache.stormcrawler.opensearch.IndexCreation;
-import org.apache.stormcrawler.opensearch.OpenSearchConnection;
-import org.apache.stormcrawler.opensearch.WaitAckCache;
-import org.apache.stormcrawler.persistence.AbstractStatusUpdaterBolt;
-import org.apache.stormcrawler.persistence.Status;
-import org.apache.stormcrawler.util.ConfUtils;
-import org.apache.stormcrawler.util.URLPartitioner;
-import org.opensearch.client.opensearch.core.BulkRequest;
-import org.opensearch.client.opensearch.core.BulkResponse;
-import org.opensearch.client.opensearch.core.bulk.BulkOperation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Simple bolt which stores the status of URLs into OpenSearch. Takes the tuples coming from the
- * 'status' stream. To be used in combination with a Spout to read from the index.
- */
-public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt
- implements AsyncBulkProcessor.Listener {
-
- private static final Logger LOG = LoggerFactory.getLogger(StatusUpdaterBolt.class);
-
- private String OSBoltType = "status";
-
- private static final String OSStatusIndexNameParamName =
- Constants.PARAMPREFIX + "%s.index.name";
- private static final String OSStatusRoutingParamName = Constants.PARAMPREFIX + "%s.routing";
- private static final String OSStatusRoutingFieldParamName =
- Constants.PARAMPREFIX + "%s.routing.fieldname";
-
- private boolean routingFieldNameInMetadata = false;
-
- private String indexName;
-
- private URLPartitioner partitioner;
-
- /** whether to apply the same partitioning logic used for politeness for routing, e.g byHost */
- private boolean doRouting;
-
- /** Store the key used for routing explicitly as a field in metadata * */
- private String fieldNameForRoutingKey = null;
-
- private OpenSearchConnection connection;
-
- private WaitAckCache waitAck;
-
- private ScopedCounter eventCounter;
-
- private ScopedReducedMetric receivedPerSecMetrics;
-
- public StatusUpdaterBolt() {
- super();
- }
-
- /**
- * Loads the configuration using a substring different from the default value 'status' in order
- * to distinguish it from the spout configurations
- */
- public StatusUpdaterBolt(String boltType) {
- super();
- OSBoltType = boltType;
- }
-
- @Override
- public void prepare(
- Map<String, Object> stormConf, TopologyContext context, OutputCollector collector) {
-
- super.prepare(stormConf, context, collector);
-
- indexName =
- ConfUtils.getString(
- stormConf,
- String.format(
- Locale.ROOT,
- StatusUpdaterBolt.OSStatusIndexNameParamName,
- OSBoltType),
- "status");
-
- doRouting =
- ConfUtils.getBoolean(
- stormConf,
- String.format(
- Locale.ROOT,
- StatusUpdaterBolt.OSStatusRoutingParamName,
- OSBoltType),
- false);
-
- partitioner = new URLPartitioner();
- partitioner.configure(stormConf);
-
- fieldNameForRoutingKey =
- ConfUtils.getString(
- stormConf,
- String.format(
- Locale.ROOT,
- StatusUpdaterBolt.OSStatusRoutingFieldParamName,
- OSBoltType));
- if (StringUtils.isNotBlank(fieldNameForRoutingKey)) {
- if (fieldNameForRoutingKey.startsWith("metadata.")) {
- routingFieldNameInMetadata = true;
- fieldNameForRoutingKey = fieldNameForRoutingKey.substring("metadata.".length());
- }
- // periods are not allowed in - replace with %2E
- fieldNameForRoutingKey = fieldNameForRoutingKey.replaceAll("\\.", "%2E");
- }
-
- int metrics_time_bucket_secs = 30;
-
- // benchmarking - average number of items received back from OpenSearch per second
- this.receivedPerSecMetrics =
- CrawlerMetrics.registerPerSecMetric(
- context, stormConf, "average_persec", metrics_time_bucket_secs);
-
- // eventCounter MUST be registered before WaitAckCache — the eviction lambda captures it
- this.eventCounter =
- CrawlerMetrics.registerCounter(
- context, stormConf, "counters", metrics_time_bucket_secs);
-
- String defaultSpec =
- String.format(
- Locale.ROOT,
- "expireAfterWrite=%ds",
- ConfUtils.getInt(stormConf, "topology.message.timeout.secs", 300));
-
- String waitAckSpec =
- ConfUtils.getString(stormConf, "opensearch.status.waitack.cache.spec", defaultSpec);
-
- waitAck =
- new WaitAckCache(
- waitAckSpec,
- LOG,
- t -> {
- eventCounter.scope("purged").incrBy(1);
- collector.fail(t);
- });
- CrawlerMetrics.registerGauge(
- context, stormConf, "waitAck", waitAck::estimatedSize, metrics_time_bucket_secs);
-
- try {
- connection = OpenSearchConnection.getConnection(stormConf, OSBoltType, this);
- } catch (Exception e1) {
- LOG.error("Can't connect to OpenSearch", e1);
- throw new RuntimeException(e1);
- }
-
- // use the default status schema if none has been specified
- try {
- IndexCreation.checkOrCreateIndex(connection.getClient(), indexName, OSBoltType, LOG);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void cleanup() {
- waitAck.shutdown();
- if (connection == null) {
- return;
- }
- connection.close();
- connection = null;
- }
-
- @Override
- public void store(
- String url, Status status, Metadata metadata, Optional<Date> nextFetch, Tuple tuple)
- throws Exception {
-
- String documentID = getDocumentID(metadata, url);
-
- boolean isAlreadySentAndDiscovered =
- status.equals(Status.DISCOVERED) && waitAck.contains(documentID);
-
- if (isAlreadySentAndDiscovered) {
- // if this object is discovered - adding another version of it
- // won't make any difference
- LOG.debug(
- "Already being sent to OpenSearch {} with status {} and ID {}",
- url,
- status,
- documentID);
- // ack straight away!
- eventCounter.scope("skipped").incrBy(1);
- super.ack(tuple, url);
- return;
- }
-
- Map<String, Object> doc = new HashMap<>();
- doc.put("url", url);
- doc.put("status", status.name());
-
- Map<String, Object> metadataMap = new HashMap<>();
- for (String mdKey : metadata.keySet()) {
- String[] values = metadata.getValues(mdKey);
- // periods are not allowed - replace with %2E
- mdKey = mdKey.replaceAll("\\.", "%2E");
- metadataMap.put(mdKey, Arrays.asList(values));
- }
-
- String partitionKey = partitioner.getPartition(url, metadata);
- if (partitionKey == null) {
- partitionKey = "_DEFAULT_";
- }
-
- // send a tuple on the queue stream in case a bolt
- // wants to handle it
- super.collector.emit(
- org.apache.stormcrawler.Constants.QUEUE_STREAM_NAME,
- new Values(partitionKey, metadata));
-
- // store routing key in metadata?
- if (StringUtils.isNotBlank(fieldNameForRoutingKey) && routingFieldNameInMetadata) {
- metadataMap.put(fieldNameForRoutingKey, partitionKey);
- }
-
- doc.put("metadata", metadataMap);
-
- // store routing key outside metadata?
- if (StringUtils.isNotBlank(fieldNameForRoutingKey) && !routingFieldNameInMetadata) {
- doc.put(fieldNameForRoutingKey, partitionKey);
- }
-
- if (nextFetch.isPresent()) {
- doc.put("nextFetchDate", nextFetch.get().toInstant().toString());
- }
- // check that we don't overwrite an existing entry
- // When create is used, the index operation will fail if a document
- // by that id already exists in the index.
- final boolean create = status.equals(Status.DISCOVERED);
- final String targetIndex = getIndexName(metadata);
- final String routing = doRouting ? partitionKey : null;
-
- BulkOperation op;
- if (create) {
- op =
- BulkOperation.of(
- b ->
- b.create(
- c -> {
- c.index(targetIndex).id(documentID).document(doc);
- if (routing != null) {
- c.routing(routing);
- }
- return c;
- }));
- } else {
- op =
- BulkOperation.of(
- b ->
- b.index(
- idx -> {
- idx.index(targetIndex).id(documentID).document(doc);
- if (routing != null) {
- idx.routing(routing);
- }
- return idx;
- }));
- }
-
- waitAck.addTuple(documentID, tuple);
-
- LOG.debug("Sending to OpenSearch buffer {} with ID {}", url, documentID);
-
- connection.addToProcessor(op);
- }
-
- @Override
- public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
- LOG.debug("afterBulk [{}] with {} responses", executionId, request.operations().size());
- eventCounter.scope("bulks_received").incrBy(1);
- eventCounter.scope("bulk_msec").incrBy(response.took());
- eventCounter.scope("received").incrBy(request.operations().size());
- receivedPerSecMetrics.scope("received").update(request.operations().size());
-
- waitAck.processBulkResponse(
- response,
- executionId,
- eventCounter,
- (id, tuple, selected) -> {
- if (!selected.failed()) {
- String url = tuple.getStringByField("url");
- LOG.debug("Acked {} with ID {}", url, id);
- eventCounter.scope("acked").incrBy(1);
- super.ack(tuple, url);
- } else {
- eventCounter.scope("failed").incrBy(1);
- collector.fail(tuple);
- }
- });
- }
-
- @Override
- public void afterBulk(long executionId, BulkRequest request, Throwable throwable) {
- eventCounter.scope("bulks_received").incrBy(1);
- eventCounter.scope("received").incrBy(request.operations().size());
- receivedPerSecMetrics.scope("received").update(request.operations().size());
-
- waitAck.processFailedBulk(
- request,
- executionId,
- throwable,
- t -> {
- eventCounter.scope("failed").incrBy(1);
- collector.fail(t);
- });
- }
-
- @Override
- public void beforeBulk(long executionId, BulkRequest request) {
- LOG.debug("beforeBulk {} with {} actions", executionId, request.operations().size());
- eventCounter.scope("bulks_sent").incrBy(1);
- }
-
- /**
- * Must be overridden for implementing custom index names based on some metadata information By
- * Default, indexName coming from config is used
- */
- protected String getIndexName(Metadata m) {
- return indexName;
- }
-}
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/DelegateRefresherTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/DelegateRefresherTest.java
deleted file mode 100644
index 67bcca62..00000000
--- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/DelegateRefresherTest.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertInstanceOf;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.stormcrawler.JSONResource;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.filtering.URLFilter;
-import org.apache.stormcrawler.parse.ParseFilter;
-import org.apache.stormcrawler.parse.ParseResult;
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-import org.junit.jupiter.api.Test;
-import org.w3c.dom.DocumentFragment;
-
-class DelegateRefresherTest {
-
- private static final ObjectMapper MAPPER = new ObjectMapper();
-
- /** Minimal URLFilter + JSONResource implementation for testing. */
- public static class StubURLFilter extends URLFilter implements JSONResource {
-
- public final AtomicBoolean configured = new AtomicBoolean(false);
-
- @Override
- public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode params) {
- configured.set(true);
- }
-
- @Override
- public @Nullable String filter(
- @Nullable URL sourceUrl,
- @Nullable Metadata sourceMetadata,
- @NotNull String urlToFilter) {
- return urlToFilter;
- }
-
- @Override
- public String getResourceFile() {
- return "stub.json";
- }
-
- @Override
- public void loadJSONResources(InputStream inputStream) throws IOException {}
- }
-
- /** Minimal ParseFilter + JSONResource implementation for testing. */
- public static class StubParseFilter extends ParseFilter implements JSONResource {
-
- public final AtomicBoolean configured = new AtomicBoolean(false);
-
- @Override
- public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode params) {
- configured.set(true);
- }
-
- @Override
- public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) {}
-
- @Override
- public String getResourceFile() {
- return "stub.json";
- }
-
- @Override
- public void loadJSONResources(InputStream inputStream) throws IOException {}
- }
-
- /** A URLFilter that does NOT implement JSONResource. */
- public static class NonJsonResourceURLFilter extends URLFilter {
-
- @Override
- public @Nullable String filter(
- @Nullable URL sourceUrl,
- @Nullable Metadata sourceMetadata,
- @NotNull String urlToFilter) {
- return urlToFilter;
- }
- }
-
- /** Not a URLFilter at all. */
- public static class NotAFilter {}
-
- private JsonNode buildParams(String delegateClass) {
- return buildParams(delegateClass, 600);
- }
-
- private JsonNode buildParams(String delegateClass, int refreshRate) {
- ObjectNode delegate = MAPPER.createObjectNode();
- delegate.put("class", delegateClass);
- delegate.set("params", MAPPER.createObjectNode());
-
- ObjectNode params = MAPPER.createObjectNode();
- params.set("delegate", delegate);
- params.put("refresh", refreshRate);
- return params;
- }
-
- @Test
- void loadsURLFilterDelegate() {
- JsonNode params = buildParams(StubURLFilter.class.getName());
- Map<String, Object> conf = new HashMap<>();
-
- DelegateRefresher<URLFilter> refresher =
- new DelegateRefresher<>(
- URLFilter.class, conf, params, (d, c, p) -> d.configure(c, p));
-
- try {
- assertNotNull(refresher.getDelegate());
- assertInstanceOf(StubURLFilter.class, refresher.getDelegate());
- assertTrue(((StubURLFilter) refresher.getDelegate()).configured.get());
- } finally {
- refresher.cleanup();
- }
- }
-
- @Test
- void loadsParseFilterDelegate() {
- JsonNode params = buildParams(StubParseFilter.class.getName());
- Map<String, Object> conf = new HashMap<>();
-
- DelegateRefresher<ParseFilter> refresher =
- new DelegateRefresher<>(
- ParseFilter.class, conf, params, (d, c, p) -> d.configure(c, p));
-
- try {
- assertNotNull(refresher.getDelegate());
- assertInstanceOf(StubParseFilter.class, refresher.getDelegate());
- assertTrue(((StubParseFilter) refresher.getDelegate()).configured.get());
- } finally {
- refresher.cleanup();
- }
- }
-
- @Test
- void delegateFilterActuallyWorks() {
- JsonNode params = buildParams(StubURLFilter.class.getName());
- Map<String, Object> conf = new HashMap<>();
-
- DelegateRefresher<URLFilter> refresher =
- new DelegateRefresher<>(
- URLFilter.class, conf, params, (d, c, p) -> d.configure(c, p));
-
- try {
- String result = refresher.getDelegate().filter(null, null, "http://example.com");
- assertEquals("http://example.com", result);
- } finally {
- refresher.cleanup();
- }
- }
-
- @Test
- void throwsWhenDelegateNodeMissing() {
- ObjectNode params = MAPPER.createObjectNode();
- // no "delegate" key
- Map<String, Object> conf = new HashMap<>();
-
- assertThrows(
- RuntimeException.class,
- () ->
- new DelegateRefresher<>(
- URLFilter.class, conf, params, (d, c, p) -> d.configure(c, p)));
- }
-
- @Test
- void throwsWhenClassMissing() {
- ObjectNode delegate = MAPPER.createObjectNode();
- // no "class" key
- ObjectNode params = MAPPER.createObjectNode();
- params.set("delegate", delegate);
- Map<String, Object> conf = new HashMap<>();
-
- assertThrows(
- RuntimeException.class,
- () ->
- new DelegateRefresher<>(
- URLFilter.class, conf, params, (d, c, p) -> d.configure(c, p)));
- }
-
- @Test
- void throwsWhenClassDoesNotExtendBaseType() {
- JsonNode params = buildParams(NotAFilter.class.getName());
- Map<String, Object> conf = new HashMap<>();
-
- RuntimeException ex =
- assertThrows(
- RuntimeException.class,
- () ->
- new DelegateRefresher<>(
- URLFilter.class,
- conf,
- params,
- (d, c, p) -> d.configure(c, p)));
- assertTrue(ex.getMessage().contains("does not extend"));
- }
-
- @Test
- void throwsWhenClassDoesNotImplementJSONResource() {
- JsonNode params = buildParams(NonJsonResourceURLFilter.class.getName());
- Map<String, Object> conf = new HashMap<>();
-
- RuntimeException ex =
- assertThrows(
- RuntimeException.class,
- () ->
- new DelegateRefresher<>(
- URLFilter.class,
- conf,
- params,
- (d, c, p) -> d.configure(c, p)));
- assertTrue(ex.getMessage().contains("does not implement JSONResource"));
- }
-
- @Test
- void cleanupIsIdempotent() {
- JsonNode params = buildParams(StubURLFilter.class.getName());
- Map<String, Object> conf = new HashMap<>();
-
- DelegateRefresher<URLFilter> refresher =
- new DelegateRefresher<>(
- URLFilter.class, conf, params, (d, c, p) -> d.configure(c, p));
-
- // calling cleanup twice should not throw
- refresher.cleanup();
- refresher.cleanup();
- }
-}
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java
deleted file mode 100644
index bb484e84..00000000
--- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.util.List;
-import org.apache.hc.client5.http.auth.AuthScope;
-import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
-import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
-import org.apache.hc.core5.http.HttpHost;
-import org.junit.jupiter.api.Test;
-
-class OpenSearchConnectionCredentialsTest {
-
- private static BasicCredentialsProvider providerFor(List<HttpHost> hosts) {
- final BasicCredentialsProvider provider = new BasicCredentialsProvider();
- final UsernamePasswordCredentials credentials =
- new UsernamePasswordCredentials("crawler", "s3cret".toCharArray());
- for (AuthScope scope : OpenSearchConnection.credentialScopes(hosts)) {
- provider.setCredentials(scope, credentials);
- }
- return provider;
- }
-
- @Test
- void scopesCoverTheConfiguredHostsAndPorts() {
- final List<AuthScope> scopes =
- OpenSearchConnection.credentialScopes(
- List.of(
- new HttpHost("https", "opensearch1.example.org", 9200),
- new HttpHost("https", "opensearch2.example.org", 9201)));
- assertEquals(
- List.of(
- new AuthScope("https", "opensearch1.example.org", 9200, null, null),
- new AuthScope("https", "opensearch2.example.org", 9201, null, null)),
- scopes);
- }
-
- @Test
- void duplicateAddressesGiveOneScope() {
- final List<AuthScope> scopes =
- OpenSearchConnection.credentialScopes(
- List.of(
- new HttpHost("https", "opensearch1.example.org", 9200),
- new HttpHost("https", "OpenSearch1.example.org", 9200)));
- assertEquals(1, scopes.size());
- }
-
- @Test
- void noAddressGivesNoScope() {
- assertTrue(OpenSearchConnection.credentialScopes(List.of()).isEmpty());
- }
-
- @Test
- void credentialsAreOnlyGivenToTheConfiguredHosts() {
- final BasicCredentialsProvider provider =
- providerFor(List.of(new HttpHost("https", "opensearch1.example.org", 9200)));
-
- assertNotNull(
- provider.getCredentials(request("https", "opensearch1.example.org", 9200), null));
- // another node of the cluster which is not listed in the addresses
- assertNull(provider.getCredentials(request("https", "10.0.0.12", 9200), null));
- // same host, another port
- assertNull(
- provider.getCredentials(request("https", "opensearch1.example.org", 9300), null));
- assertNull(provider.getCredentials(request("https", "other.example.org", 9200), null));
- }
-
- @Test
- void credentialsAreMatchedOnTheScheme() {
- final BasicCredentialsProvider provider =
- providerFor(List.of(new HttpHost("https", "opensearch1.example.org", 9200)));
- assertNull(provider.getCredentials(request("http", "opensearch1.example.org", 9200), null));
- }
-
- /** The scope HttpClient looks the credentials up with for a request to the given node. */
- private static AuthScope request(String scheme, String host, int port) {
- return new AuthScope(new HttpHost(scheme, host, port), null, "Basic");
- }
-
- @Test
- void plainHttpToARemoteHostIsReported() {
- final HttpHost remote = new HttpHost("http", "opensearch1.example.org", 9200);
- final List<HttpHost> plain =
- OpenSearchConnection.plainHttpHosts(
- List.of(
- remote,
- new HttpHost("https", "opensearch2.example.org", 9200),
- new HttpHost("http", "localhost", 9200),
- new HttpHost("http", "127.0.0.1", 9200),
- new HttpHost("http", "[::1]", 9200)));
- assertEquals(List.of(remote), plain);
- }
-
- @Test
- void loopbackAddresses() {
- assertTrue(OpenSearchConnection.isLoopback("localhost"));
- assertTrue(OpenSearchConnection.isLoopback("LOCALHOST"));
- assertTrue(OpenSearchConnection.isLoopback("127.0.0.1"));
- assertTrue(OpenSearchConnection.isLoopback("[::1]"));
- assertTrue(OpenSearchConnection.isLoopback("0:0:0:0:0:0:0:1"));
- assertFalse(OpenSearchConnection.isLoopback("opensearch1.example.org"));
- assertFalse(OpenSearchConnection.isLoopback("10.0.0.12"));
- assertFalse(OpenSearchConnection.isLoopback("localhost.example.org"));
- assertFalse(OpenSearchConnection.isLoopback("127.example.org"));
- assertTrue(OpenSearchConnection.isLoopback("127.1.2.3"));
- }
-}
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java
deleted file mode 100644
index 3bd2dc6d..00000000
--- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java
+++ /dev/null
@@ -1,336 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch;
-
-import static org.awaitility.Awaitility.await;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertSame;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.TimeUnit;
-import org.apache.storm.tuple.Tuple;
-import org.apache.stormcrawler.metrics.ScopedCounter;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.opensearch.client.opensearch._types.ErrorCause;
-import org.opensearch.client.opensearch.core.BulkRequest;
-import org.opensearch.client.opensearch.core.BulkResponse;
-import org.opensearch.client.opensearch.core.bulk.BulkOperation;
-import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
-import org.opensearch.client.opensearch.core.bulk.OperationType;
-import org.slf4j.LoggerFactory;
-
-class WaitAckCacheTest {
-
- private WaitAckCache cache;
- private List<Tuple> evicted;
- private List<Tuple> acked;
- private List<Tuple> failed;
-
- @BeforeEach
- void setUp() {
- evicted = new CopyOnWriteArrayList<>();
- acked = new ArrayList<>();
- failed = new ArrayList<>();
- cache = new WaitAckCache(LoggerFactory.getLogger(WaitAckCacheTest.class), evicted::add);
- }
-
- private Tuple mockTuple(String url) {
- Tuple t = mock(Tuple.class);
- when(t.getValueByField("url")).thenReturn(url);
- when(t.getStringByField("url")).thenReturn(url);
- return t;
- }
-
- private static BulkResponseItem successItem(String docId) {
- return BulkResponseItem.of(
- b -> b.id(docId).index("index").status(200).operationType(OperationType.Index));
- }
-
- private static BulkResponseItem failedItem(String docId, int status) {
- return BulkResponseItem.of(
- b ->
- b.id(docId)
- .index("index")
- .status(status)
- .operationType(OperationType.Index)
- .error(
- ErrorCause.of(
- e -> e.type("test_error").reason("test failure"))));
- }
-
- private static BulkResponse bulkResponse(BulkResponseItem... items) {
- boolean hasErrors = false;
- for (BulkResponseItem item : items) {
- if (item.error() != null) {
- hasErrors = true;
- break;
- }
- }
- final boolean errors = hasErrors;
- return BulkResponse.of(b -> b.took(10).errors(errors).items(List.of(items)));
- }
-
- @Test
- void addAndContains() {
- Tuple t = mockTuple("http://example.com");
- assertFalse(cache.contains("doc1"));
-
- cache.addTuple("doc1", t);
- assertTrue(cache.contains("doc1"));
- assertEquals(1, cache.estimatedSize());
- }
-
- @Test
- void invalidateRemovesEntry() {
- Tuple t = mockTuple("http://example.com");
- cache.addTuple("doc1", t);
- assertTrue(cache.contains("doc1"));
-
- cache.invalidate("doc1");
- assertFalse(cache.contains("doc1"));
- }
-
- @Test
- void processBulkResponse_successfulItem_ackedViaTupleAction() {
- Tuple t = mockTuple("http://example.com");
- cache.addTuple("doc1", t);
-
- BulkResponse response = bulkResponse(successItem("doc1"));
-
- cache.processBulkResponse(
- response,
- 1L,
- null,
- (id, tuple, selected) -> {
- if (!selected.failed()) {
- acked.add(tuple);
- } else {
- failed.add(tuple);
- }
- });
-
- assertEquals(1, acked.size());
- assertEquals(0, failed.size());
- assertSame(t, acked.get(0));
- assertFalse(cache.contains("doc1"));
- }
-
- @Test
- void processBulkResponse_failedItem_failedViaTupleAction() {
- Tuple t = mockTuple("http://example.com");
- cache.addTuple("doc1", t);
-
- BulkResponse response = bulkResponse(failedItem("doc1", 500));
-
- cache.processBulkResponse(
- response,
- 1L,
- null,
- (id, tuple, selected) -> {
- if (!selected.failed()) {
- acked.add(tuple);
- } else {
- failed.add(tuple);
- }
- });
-
- assertEquals(0, acked.size());
- assertEquals(1, failed.size());
- assertSame(t, failed.get(0));
- }
-
- @Test
- void processBulkResponse_conflictIsNotAFailure() {
- Tuple t = mockTuple("http://example.com");
- cache.addTuple("doc1", t);
-
- ScopedCounter counter = mock(ScopedCounter.class);
- ScopedCounter.CountHandle handle = mock(ScopedCounter.CountHandle.class);
- when(counter.scope("doc_conflicts")).thenReturn(handle);
-
- BulkResponse response = bulkResponse(failedItem("doc1", 409));
-
- cache.processBulkResponse(
- response,
- 1L,
- counter,
- (id, tuple, selected) -> {
- if (!selected.failed()) {
- acked.add(tuple);
- } else {
- failed.add(tuple);
- }
- });
-
- assertEquals(1, acked.size());
- assertEquals(0, failed.size());
- verify(handle).incrBy(1);
- }
-
- @Test
- void processBulkResponse_multipleTuplesForSameDocId() {
- Tuple t1 = mockTuple("http://example.com/1");
- Tuple t2 = mockTuple("http://example.com/2");
- cache.addTuple("doc1", t1);
- cache.addTuple("doc1", t2);
-
- BulkResponse response = bulkResponse(successItem("doc1"));
-
- cache.processBulkResponse(response, 1L, null, (id, tuple, selected) -> acked.add(tuple));
-
- assertEquals(2, acked.size());
- assertTrue(acked.contains(t1));
- assertTrue(acked.contains(t2));
- }
-
- @Test
- void processBulkResponse_duplicateDocIdInBulk_prefersSuccess() {
- // https://github.com/apache/stormcrawler/issues/832
- Tuple t = mockTuple("http://example.com");
- cache.addTuple("doc1", t);
-
- BulkResponse response = bulkResponse(failedItem("doc1", 500), successItem("doc1"));
-
- cache.processBulkResponse(
- response,
- 1L,
- null,
- (id, tuple, selected) -> {
- if (!selected.failed()) {
- acked.add(tuple);
- } else {
- failed.add(tuple);
- }
- });
-
- assertEquals(1, acked.size());
- assertEquals(0, failed.size());
- }
-
- @Test
- void processFailedBulk_failsAllMatchingTuples() {
- Tuple t1 = mockTuple("http://example.com/1");
- Tuple t2 = mockTuple("http://example.com/2");
- cache.addTuple("doc1", t1);
- cache.addTuple("doc2", t2);
-
- BulkRequest request =
- BulkRequest.of(
- b ->
- b.operations(
- BulkOperation.of(
- o -> o.delete(d -> d.index("index").id("doc1"))),
- BulkOperation.of(
- o -> o.delete(d -> d.index("index").id("doc2")))));
-
- cache.processFailedBulk(request, 1L, new Exception("connection lost"), failed::add);
-
- assertEquals(2, failed.size());
- assertTrue(failed.contains(t1));
- assertTrue(failed.contains(t2));
- assertFalse(cache.contains("doc1"));
- assertFalse(cache.contains("doc2"));
- }
-
- @Test
- void processFailedBulk_ignoresMissingIds() {
- Tuple t = mockTuple("http://example.com");
- cache.addTuple("doc1", t);
-
- BulkRequest request =
- BulkRequest.of(
- b ->
- b.operations(
- BulkOperation.of(
- o ->
- o.delete(
- d ->
- d.index("index")
- .id(
- "doc_unknown")))));
-
- cache.processFailedBulk(request, 1L, new Exception("test"), failed::add);
-
- assertEquals(0, failed.size());
- // doc1 should still be in cache since it wasn't in the failed request
- assertTrue(cache.contains("doc1"));
- }
-
- @Test
- void eviction_failsTuplesOnExpiry() {
- cache =
- new WaitAckCache(
- "expireAfterWrite=1s",
- LoggerFactory.getLogger(WaitAckCacheTest.class),
- evicted::add);
- Tuple t = mockTuple("http://example.com");
- cache.addTuple("doc1", t);
-
- // Force cache maintenance after expiry by doing a contains() check
- // which accesses the cache and triggers Caffeine's cleanup
- await().atMost(5, TimeUnit.SECONDS)
- .pollInterval(200, TimeUnit.MILLISECONDS)
- .untilAsserted(
- () -> {
- // contains() accesses the cache which triggers cleanup
- cache.contains("doc1");
- // also try adding and invalidating a dummy entry to force maintenance
- Tuple dummy = mockTuple("http://dummy");
- cache.addTuple("_probe_", dummy);
- cache.invalidate("_probe_");
- assertFalse(evicted.isEmpty(), "Eviction callback should have fired");
- });
-
- assertTrue(evicted.contains(t));
- }
-
- @Test
- void processBulkResponse_multipleDocIds() {
- Tuple t1 = mockTuple("http://example.com/1");
- Tuple t2 = mockTuple("http://example.com/2");
- cache.addTuple("doc1", t1);
- cache.addTuple("doc2", t2);
-
- BulkResponse response = bulkResponse(successItem("doc1"), failedItem("doc2", 500));
-
- cache.processBulkResponse(
- response,
- 1L,
- null,
- (id, tuple, selected) -> {
- if (!selected.failed()) {
- acked.add(tuple);
- } else {
- failed.add(tuple);
- }
- });
-
- assertEquals(1, acked.size());
- assertSame(t1, acked.get(0));
- assertEquals(1, failed.size());
- assertSame(t2, failed.get(0));
- }
-}
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
deleted file mode 100644
index 44150ea6..00000000
--- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.bolt;
-
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.junit.jupiter.Testcontainers;
-import org.testcontainers.utility.DockerImageName;
-
-@Testcontainers(disabledWithoutDocker = true)
-public abstract class AbstractOpenSearchTest {
-
- private static final String OPENSEARCH_VERSION =
- System.getProperty("opensearch-version", "2.19.5");
-
- public static final String PASSWORD = "This1sAPassw0rd";
-
- protected GenericContainer<?> opensearchContainer =
- new GenericContainer<>(
- DockerImageName.parse(
- "opensearchproject/opensearch:" + OPENSEARCH_VERSION))
- .withExposedPorts(9200)
- .withEnv("plugins.security.disabled", "true")
- .withEnv("discovery.type", "single-node")
- .withEnv("OPENSEARCH_JAVA_OPTS", "-Xms512m -Xmx512m")
- .withEnv("OPENSEARCH_INITIAL_ADMIN_PASSWORD", PASSWORD);
-
- @BeforeEach
- void init() {
- opensearchContainer.start();
- }
-
- @AfterEach
- void close() {
- opensearchContainer.close();
- }
-}
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/IndexerBoltTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/IndexerBoltTest.java
deleted file mode 100644
index aa953a28..00000000
--- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/IndexerBoltTest.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.bolt;
-
-import static org.awaitility.Awaitility.await;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.tuple.Tuple;
-import org.apache.stormcrawler.Constants;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.TestOutputCollector;
-import org.apache.stormcrawler.TestUtil;
-import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-class IndexerBoltTest extends AbstractOpenSearchTest {
-
- private IndexerBolt bolt;
-
- protected TestOutputCollector output;
-
- private static final Logger LOG = LoggerFactory.getLogger(IndexerBoltTest.class);
-
- private static ExecutorService executorService;
-
- @BeforeAll
- static void beforeClass() {
- executorService = Executors.newFixedThreadPool(2);
- }
-
- @AfterAll
- static void afterClass() {
- executorService.shutdown();
- executorService = null;
- }
-
- @BeforeEach
- void setupIndexerBolt() {
- bolt = new IndexerBolt("content");
- // give the indexer the port for connecting to OpenSearch
- final String host = opensearchContainer.getHost();
- final Integer port = opensearchContainer.getFirstMappedPort();
- final Map<String, Object> conf = new HashMap<>();
- conf.put(AbstractIndexerBolt.urlFieldParamName, "url");
- conf.put(AbstractIndexerBolt.canonicalMetadataParamName, "canonical");
- conf.put("opensearch.indexer.addresses", host + ":" + port);
- output = new TestOutputCollector();
- bolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
- }
-
- @AfterEach
- void close() {
- LOG.info("Closing indexer bolt and Opensearch container");
- super.close();
- bolt.cleanup();
- output = null;
- }
-
- private void index(String url, String text, Metadata metadata) {
- Tuple tuple = mock(Tuple.class);
- when(tuple.getStringByField("text")).thenReturn(text);
- when(tuple.getStringByField("url")).thenReturn(url);
- when(tuple.getValueByField("metadata")).thenReturn(metadata);
- bolt.execute(tuple);
- }
-
- private int lastIndex(String url, String text, Metadata metadata, long timeoutInMs)
- throws ExecutionException, InterruptedException, TimeoutException {
- var oldSize = output.getEmitted(Constants.StatusStreamName).size();
- index(url, text, metadata);
- return executorService
- .submit(
- () -> {
- await().atMost(timeoutInMs, TimeUnit.MILLISECONDS)
- .until(
- () ->
- output.getEmitted(Constants.StatusStreamName)
- .size()
- > oldSize);
- return output.getEmitted(Constants.StatusStreamName).size();
- })
- .get(timeoutInMs, TimeUnit.MILLISECONDS);
- }
-
- @Test
- @Timeout(value = 2, unit = TimeUnit.MINUTES)
- // https://github.com/apache/stormcrawler/issues/832
- void simultaneousCanonicals()
- throws ExecutionException, InterruptedException, TimeoutException {
- Metadata m1 = new Metadata();
- String url =
- "https://www.obozrevatel.com/ukr/dnipro/city/u-dnipri-ta-oblasti-ogolosili-shtormove-poperedzhennya.htm";
- m1.addValue("canonical", url);
- Metadata m2 = new Metadata();
- String url2 =
- "https://www.obozrevatel.com/ukr/dnipro/city/u-dnipri-ta-oblasti-ogolosili-shtormove-poperedzhennya/amp.htm";
- m2.addValue("canonical", url);
- index(url, "", m1);
- lastIndex(url2, "", m2, 10_000);
- // should be two in status output
- assertEquals(2, output.getEmitted(Constants.StatusStreamName).size());
- // and 2 acked
- assertEquals(2, output.getAckedTuples().size());
- // TODO check output in Opensearch?
- }
-}
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java
deleted file mode 100644
index 296c58dd..00000000
--- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.bolt;
-
-import static org.awaitility.Awaitility.await;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import org.apache.hc.core5.http.HttpHost;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.tuple.Tuple;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.TestOutputCollector;
-import org.apache.stormcrawler.TestUtil;
-import org.apache.stormcrawler.opensearch.persistence.QueueBolt;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.Timeout;
-import org.opensearch.client.json.jackson.JacksonJsonpMapper;
-import org.opensearch.client.opensearch.OpenSearchClient;
-import org.opensearch.client.opensearch.core.GetResponse;
-import org.opensearch.client.transport.OpenSearchTransport;
-import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-class QueueBoltTest extends AbstractOpenSearchTest {
-
- private QueueBolt bolt;
-
- protected TestOutputCollector output;
-
- protected OpenSearchClient client;
-
- private OpenSearchTransport transport;
-
- private static final Logger LOG = LoggerFactory.getLogger(QueueBoltTest.class);
-
- private static ExecutorService executorService;
-
- @BeforeAll
- static void beforeClass() {
- executorService = Executors.newFixedThreadPool(2);
- }
-
- @AfterAll
- static void afterClass() {
- executorService.shutdown();
- executorService = null;
- }
-
- @BeforeEach
- void setupQueueBolt() throws IOException {
- transport =
- ApacheHttpClient5TransportBuilder.builder(
- new HttpHost(
- "http",
- opensearchContainer.getHost(),
- opensearchContainer.getMappedPort(9200)))
- .setMapper(new JacksonJsonpMapper())
- .build();
- client = new OpenSearchClient(transport);
- output = new TestOutputCollector();
- bolt = prepareQueueBolt();
- }
-
- private QueueBolt prepareQueueBolt() {
- QueueBolt queueBolt = new QueueBolt();
- Map<String, Object> conf = new HashMap<>();
- conf.put(
- "opensearch.queues.addresses",
- opensearchContainer.getHost() + ":" + opensearchContainer.getFirstMappedPort());
- queueBolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
- return queueBolt;
- }
-
- @AfterEach
- void close() {
- LOG.info("Closing queue bolt and Opensearch container");
- super.close();
- bolt.cleanup();
- output = null;
- try {
- transport.close();
- } catch (IOException e) {
- }
- }
-
- private Future<Integer> store(String key, Metadata metadata) {
- Tuple tuple = mock(Tuple.class);
- when(tuple.getStringByField("key")).thenReturn(key);
- when(tuple.getValueByField("metadata")).thenReturn(metadata);
- bolt.execute(tuple);
- return executorService.submit(
- () -> {
- await().atMost(30, TimeUnit.SECONDS)
- .until(() -> output.getAckedTuples().size() > 0);
- return output.getAckedTuples().size();
- });
- }
-
- private void awaitIndexed(String key) {
- String id = org.apache.commons.codec.digest.DigestUtils.sha256Hex(key);
- await().atMost(30, TimeUnit.SECONDS)
- .until(
- () -> {
- try {
- GetResponse<Map> result =
- client.get(g -> g.index("queues").id(id), Map.class);
- return result.found();
- } catch (IOException e) {
- return false;
- }
- });
- }
-
- @SuppressWarnings("unchecked")
- private Map<String, Object> getSource(String key) throws IOException {
- String id = org.apache.commons.codec.digest.DigestUtils.sha256Hex(key);
- return client.get(g -> g.index("queues").id(id), Map.class).source();
- }
-
- @Test
- @Timeout(value = 2, unit = TimeUnit.MINUTES)
- void checkQueueIndex()
- throws IOException, ExecutionException, InterruptedException, TimeoutException {
- String key = "www.url.net";
- Metadata md = new Metadata();
- md.addValue("someKey", "someValue");
- store(key, md).get(10, TimeUnit.SECONDS);
- assertEquals(1, output.getAckedTuples().size());
-
- // Wait until document is indexed in OpenSearch
- awaitIndexed(key);
-
- Map<String, Object> sourceAsMap = getSource(key);
- assertEquals(key, sourceAsMap.get("key"));
- }
-
- @Test
- @Timeout(value = 2, unit = TimeUnit.MINUTES)
- void checkExistingEntryIsNotOverwritten()
- throws IOException, ExecutionException, InterruptedException, TimeoutException {
- String key = "www.url.net";
- Metadata md = new Metadata();
- md.addValue("someKey", "someValue");
- store(key, md).get(10, TimeUnit.SECONDS);
- awaitIndexed(key);
- Object lastUpdated = getSource(key).get("lastUpdated");
-
- // a fresh bolt instance (e.g. after a worker restart) has an empty cache
- // and re-sends the same key - the resulting version conflict must be
- // handled gracefully and the existing entry left untouched
- QueueBolt restartedBolt = prepareQueueBolt();
- try {
- Tuple sameKey = mock(Tuple.class);
- when(sameKey.getStringByField("key")).thenReturn(key);
- restartedBolt.execute(sameKey);
-
- // a new key sent in the same bulk signals when the bulk has been processed
- String otherKey = "www.other.net";
- Tuple other = mock(Tuple.class);
- when(other.getStringByField("key")).thenReturn(otherKey);
- restartedBolt.execute(other);
- awaitIndexed(otherKey);
-
- Map<String, Object> sourceAsMap = getSource(key);
- assertEquals(key, sourceAsMap.get("key"));
- assertEquals(lastUpdated, sourceAsMap.get("lastUpdated"));
- } finally {
- restartedBolt.cleanup();
- }
- }
-}
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java
deleted file mode 100644
index 71fe7795..00000000
--- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.bolt;
-
-import static org.awaitility.Awaitility.await;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import org.apache.hc.core5.http.HttpHost;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.tuple.Tuple;
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.TestOutputCollector;
-import org.apache.stormcrawler.TestUtil;
-import org.apache.stormcrawler.opensearch.persistence.StatusUpdaterBolt;
-import org.apache.stormcrawler.persistence.Status;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.Timeout;
-import org.opensearch.client.json.jackson.JacksonJsonpMapper;
-import org.opensearch.client.opensearch.OpenSearchClient;
-import org.opensearch.client.opensearch.core.GetResponse;
-import org.opensearch.client.transport.OpenSearchTransport;
-import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-class StatusBoltTest extends AbstractOpenSearchTest {
-
- private StatusUpdaterBolt bolt;
-
- protected TestOutputCollector output;
-
- protected OpenSearchClient client;
-
- private OpenSearchTransport transport;
-
- private static final Logger LOG = LoggerFactory.getLogger(StatusBoltTest.class);
-
- private static ExecutorService executorService;
-
- @BeforeAll
- static void beforeClass() {
- executorService = Executors.newFixedThreadPool(2);
- }
-
- @AfterAll
- static void afterClass() {
- executorService.shutdown();
- executorService = null;
- }
-
- @BeforeEach
- void setupStatusBolt() throws IOException {
- bolt = new StatusUpdaterBolt();
- transport =
- ApacheHttpClient5TransportBuilder.builder(
- new HttpHost(
- "http",
- opensearchContainer.getHost(),
- opensearchContainer.getMappedPort(9200)))
- .setMapper(new JacksonJsonpMapper())
- .build();
- client = new OpenSearchClient(transport);
- // configure the status updater bolt
- Map<String, Object> conf = new HashMap<>();
- conf.put("opensearch.status.routing.fieldname", "metadata.key");
- conf.put(
- "opensearch.status.addresses",
- opensearchContainer.getHost() + ":" + opensearchContainer.getFirstMappedPort());
- conf.put("scheduler.class", "org.apache.stormcrawler.persistence.DefaultScheduler");
- conf.put("status.updater.cache.spec", "maximumSize=10000,expireAfterAccess=1h");
- conf.put("metadata.persist", "someKey");
- output = new TestOutputCollector();
- bolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
- }
-
- @AfterEach
- void close() {
- LOG.info("Closing updater bolt and Opensearch container");
- super.close();
- bolt.cleanup();
- output = null;
- try {
- transport.close();
- } catch (IOException e) {
- }
- }
-
- private Future<Integer> store(String url, Status status, Metadata metadata) {
- Tuple tuple = mock(Tuple.class);
- when(tuple.getValueByField("status")).thenReturn(status);
- when(tuple.getStringByField("url")).thenReturn(url);
- when(tuple.getValueByField("metadata")).thenReturn(metadata);
- bolt.execute(tuple);
- return executorService.submit(
- () -> {
- await().atMost(30, TimeUnit.SECONDS)
- .until(() -> output.getAckedTuples().size() > 0);
- return output.getAckedTuples().size();
- });
- }
-
- @Test
- @Timeout(value = 2, unit = TimeUnit.MINUTES)
- // see https://github.com/apache/stormcrawler/issues/885
- @SuppressWarnings("unchecked")
- void checkListKeyFromOpensearch()
- throws IOException, ExecutionException, InterruptedException, TimeoutException {
- String url = "https://www.url.net/something";
- Metadata md = new Metadata();
- md.addValue("someKey", "someValue");
- store(url, Status.DISCOVERED, md).get(10, TimeUnit.SECONDS);
- assertEquals(1, output.getAckedTuples().size());
- // check output in Opensearch
- String id = org.apache.commons.codec.digest.DigestUtils.sha256Hex(url);
- GetResponse<Map> result = client.get(g -> g.index("status").id(id), Map.class);
- Map<String, Object> sourceAsMap = result.source();
- final String pfield = "metadata.somekey";
- sourceAsMap = (Map<String, Object>) sourceAsMap.get("metadata");
- final var pfieldNew = pfield.substring(9);
- Object key = sourceAsMap.get(pfieldNew);
- assertTrue(key instanceof java.util.ArrayList);
- }
-
- @Test
- @Timeout(value = 2, unit = TimeUnit.MINUTES)
- void checkQueueStreamEmission()
- throws IOException, ExecutionException, InterruptedException, TimeoutException {
- String url = "https://www.url.net/something";
- Metadata md = new Metadata();
- md.addValue("someKey", "someValue");
- store(url, Status.DISCOVERED, md).get(10, TimeUnit.SECONDS);
- assertEquals(1, output.getAckedTuples().size());
-
- // Check that a tuple was emitted on the queue stream
- var emittedQueue = output.getEmitted(org.apache.stormcrawler.Constants.QUEUE_STREAM_NAME);
- assertEquals(1, emittedQueue.size());
-
- var queueTuple = emittedQueue.get(0);
- assertEquals(2, queueTuple.size());
- assertEquals("www.url.net", queueTuple.get(0));
- assertTrue(queueTuple.get(1) instanceof Metadata);
- Metadata emittedMetadata = (Metadata) queueTuple.get(1);
- assertEquals("someValue", emittedMetadata.getFirstValue("someKey"));
- }
-}
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporterTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporterTest.java
deleted file mode 100644
index 8c0c9413..00000000
--- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporterTest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.stormcrawler.opensearch.metrics;
-
-import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.MetricRegistry;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.stormcrawler.opensearch.bolt.AbstractOpenSearchTest;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.Timeout;
-
-class MetricsReporterTest extends AbstractOpenSearchTest {
-
- @Test
- @Timeout(60)
- void prepareAndReportMetrics() {
- MetricRegistry registry = new MetricRegistry();
- Counter counter = registry.counter("test.counter");
- counter.inc(42);
-
- Map<String, Object> topoConf = new HashMap<>();
- topoConf.put(
- "opensearch.metrics.addresses",
- opensearchContainer.getHost() + ":" + opensearchContainer.getFirstMappedPort());
-
- Map<String, Object> reporterConf = new HashMap<>();
- reporterConf.put("report.period", 60L);
- reporterConf.put("report.period.units", "SECONDS");
-
- MetricsReporter reporter = new MetricsReporter();
- assertDoesNotThrow(() -> reporter.prepare(registry, topoConf, reporterConf));
- assertNotNull(reporter);
- reporter.stop();
- }
-}
diff --git a/external/opensearch-java/src/test/resources/indexer.mapping b/external/opensearch-java/src/test/resources/indexer.mapping
deleted file mode 100644
index fc6eb887..00000000
--- a/external/opensearch-java/src/test/resources/indexer.mapping
+++ /dev/null
@@ -1,40 +0,0 @@
-{
- "settings": {
- "index": {
- "number_of_shards": 5,
- "number_of_replicas": 1,
- "refresh_interval": "60s"
- }
- },
- "mappings": {
- "_source": {
- "enabled": true
- },
- "properties": {
- "content": {
- "type": "text"
- },
- "description": {
- "type": "text"
- },
- "domain": {
- "type": "keyword"
- },
- "format": {
- "type": "keyword"
- },
- "keywords": {
- "type": "keyword"
- },
- "host": {
- "type": "keyword"
- },
- "title": {
- "type": "text"
- },
- "url": {
- "type": "keyword"
- }
- }
- }
-}
diff --git a/external/opensearch-java/src/test/resources/metrics.mapping b/external/opensearch-java/src/test/resources/metrics.mapping
deleted file mode 100644
index fc6ae3a0..00000000
--- a/external/opensearch-java/src/test/resources/metrics.mapping
+++ /dev/null
@@ -1,40 +0,0 @@
-{
- "index_patterns": "metrics*",
- "settings": {
- "index": {
- "number_of_shards": 1,
- "refresh_interval": "30s"
- },
- "number_of_replicas": 0
- },
- "mappings": {
- "_source": { "enabled": true },
- "properties": {
- "name": {
- "type": "keyword"
- },
- "stormId": {
- "type": "keyword"
- },
- "srcComponentId": {
- "type": "keyword"
- },
- "srcTaskId": {
- "type": "short"
- },
- "srcWorkerHost": {
- "type": "keyword"
- },
- "srcWorkerPort": {
- "type": "integer"
- },
- "timestamp": {
- "type": "date",
- "format": "date_optional_time"
- },
- "value": {
- "type": "double"
- }
- }
- }
-}
diff --git a/external/opensearch-java/src/test/resources/queues.mapping b/external/opensearch-java/src/test/resources/queues.mapping
deleted file mode 100644
index b85f33dc..00000000
--- a/external/opensearch-java/src/test/resources/queues.mapping
+++ /dev/null
@@ -1,33 +0,0 @@
-{
- "settings": {
- "index": {
- "number_of_shards": 10,
- "number_of_replicas": 1,
- "refresh_interval": "5s"
- }
- },
- "mappings": {
- "dynamic_templates": [{
- "metadata": {
- "path_match": "metadata.*",
- "match_mapping_type": "string",
- "mapping": {
- "type": "keyword"
- }
- }
- }],
- "_source": {
- "enabled": true
- },
- "properties": {
- "key": {
- "type": "keyword",
- "index": true
- },
- "lastUpdated": {
- "type": "date",
- "format": "date_optional_time"
- }
- }
- }
-}
diff --git a/external/opensearch-java/src/test/resources/status.mapping b/external/opensearch-java/src/test/resources/status.mapping
deleted file mode 100644
index e5b14fe9..00000000
--- a/external/opensearch-java/src/test/resources/status.mapping
+++ /dev/null
@@ -1,39 +0,0 @@
-{
- "settings": {
- "index": {
- "number_of_shards": 10,
- "number_of_replicas": 1,
- "refresh_interval": "5s"
- }
- },
- "mappings": {
- "dynamic_templates": [{
- "metadata": {
- "path_match": "metadata.*",
- "match_mapping_type": "string",
- "mapping": {
- "type": "keyword"
- }
- }
- }],
- "_source": {
- "enabled": true
- },
- "properties": {
- "key": {
- "type": "keyword",
- "index": true
- },
- "nextFetchDate": {
- "type": "date",
- "format": "date_optional_time"
- },
- "status": {
- "type": "keyword"
- },
- "url": {
- "type": "keyword"
- }
- }
- }
-}
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java
similarity index 100%
rename from external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java
rename to external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
index e4eec09e..c32f162f 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java
@@ -17,118 +17,55 @@
package org.apache.stormcrawler.opensearch;
-import java.io.IOException;
+import java.util.Objects;
import org.jetbrains.annotations.NotNull;
-import org.opensearch.action.DocWriteRequest;
-import org.opensearch.action.DocWriteResponse;
-import org.opensearch.action.bulk.BulkItemResponse;
-import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.core.rest.RestStatus;
-import org.opensearch.core.xcontent.ToXContent;
-import org.opensearch.core.xcontent.XContentBuilder;
+import org.jetbrains.annotations.Nullable;
+import org.opensearch.client.opensearch._types.ErrorCause;
+import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
-public final class BulkItemResponseToFailedFlag {
- @NotNull public final BulkItemResponse response;
- public final boolean failed;
- @NotNull public final String id;
+/**
+ * Wraps a {@link BulkResponseItem} with a pre-computed failure flag. A 409 (conflict) is not
+ * considered a failure — it simply indicates a document already existed when using create mode.
+ *
+ * @param response the original bulk response item
+ * @param failed whether this item represents a real failure (excludes 409 conflicts)
+ * @param id the document id from the response item
+ */
+public record BulkItemResponseToFailedFlag(
+ @NotNull BulkResponseItem response, boolean failed, @NotNull String id) {
- public BulkItemResponseToFailedFlag(@NotNull BulkItemResponse response, boolean failed) {
- this.response = response;
- this.failed = failed;
- this.id = response.getId();
+ public BulkItemResponseToFailedFlag {
+ Objects.requireNonNull(response, "response");
+ Objects.requireNonNull(id, "id");
}
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (!(o instanceof BulkItemResponseToFailedFlag)) {
- return false;
- }
-
- BulkItemResponseToFailedFlag that = (BulkItemResponseToFailedFlag) o;
-
- if (failed != that.failed) {
- return false;
- }
- if (!response.equals(that.response)) {
- return false;
- }
- return id.equals(that.id);
+ /** Constructs with id derived from the response item. */
+ public BulkItemResponseToFailedFlag(@NotNull BulkResponseItem response, boolean failed) {
+ this(
+ response,
+ failed,
+ Objects.requireNonNull(response.id(), "BulkResponseItem id must not be null"));
}
- @Override
- public int hashCode() {
- int result = response.hashCode();
- result = 31 * result + (failed ? 1 : 0);
- result = 31 * result + id.hashCode();
- return result;
+ /** Returns the error cause, or {@code null} if the item did not fail. */
+ @Nullable
+ public ErrorCause getFailedCause() {
+ return response.error();
}
- @Override
- public String toString() {
- return "BulkItemResponseToFailedFlag{"
- + "response="
- + response
- + ", failed="
- + failed
- + ", id='"
- + id
- + '\''
- + '}';
+ /** Returns a human-readable failure description, or {@code null} if the item did not fail. */
+ @Nullable
+ public String getFailure() {
+ ErrorCause error = response.error();
+ if (error == null) {
+ return null;
+ }
+ return error.reason() != null ? error.reason() : error.type();
}
- public RestStatus status() {
+ // opensearch-java: status() returns int HTTP code, not RestStatus enum
+ /** Returns the HTTP status code of this response item. */
+ public int getStatus() {
return response.status();
}
-
- public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params)
- throws IOException {
- return response.toXContent(builder, params);
- }
-
- public int getItemId() {
- return response.getItemId();
- }
-
- public DocWriteRequest.OpType getOpType() {
- return response.getOpType();
- }
-
- public String getIndex() {
- return response.getIndex();
- }
-
- public long getVersion() {
- return response.getVersion();
- }
-
- public <T extends DocWriteResponse> T getResponse() {
- return response.getResponse();
- }
-
- public boolean isFailed() {
- return response.isFailed();
- }
-
- public String getFailureMessage() {
- return response.getFailureMessage();
- }
-
- public BulkItemResponse.Failure getFailure() {
- return response.getFailure();
- }
-
- public void writeTo(StreamOutput out) throws IOException {
- response.writeTo(out);
- }
-
- public void writeThin(StreamOutput out) throws IOException {
- response.writeThin(out);
- }
-
- public boolean isFragment() {
- return response.isFragment();
- }
}
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java
index 4935bc1a..fb03a9f0 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java
@@ -20,14 +20,14 @@ package org.apache.stormcrawler.opensearch;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.ByteArrayInputStream;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import org.apache.stormcrawler.JSONResource;
-import org.opensearch.action.get.GetRequest;
-import org.opensearch.action.get.GetResponse;
-import org.opensearch.client.RequestOptions;
-import org.opensearch.client.RestHighLevelClient;
+import org.opensearch.client.json.JsonData;
+import org.opensearch.client.opensearch.OpenSearchClient;
+import org.opensearch.client.opensearch.core.GetResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,6 +38,10 @@ import org.slf4j.LoggerFactory;
* org.apache.stormcrawler.opensearch.parse.filter.JSONResourceWrapper} to eliminate duplicated
* setup/refresh/cleanup logic.
*
+ * <p>This is the opensearch-java (OpenSearch Java Client 3.x / HC5) counterpart of the class with
+ * the same name in the {@code external/opensearch} module. It uses the typed {@link
+ * OpenSearchClient} instead of the deprecated {@code RestHighLevelClient}.
+ *
* @param <T> the base type that the delegate must extend (e.g. URLFilter or ParseFilter)
*/
public class DelegateRefresher<T> {
@@ -46,7 +50,7 @@ public class DelegateRefresher<T> {
private final T delegate;
private Timer refreshTimer;
- private RestHighLevelClient osClient;
+ private OpenSearchClient osClient;
/**
* Creates a refresher by loading the delegate class from the JSON configuration.
@@ -127,13 +131,18 @@ public class DelegateRefresher<T> {
if (osClient != null) {
LOG.info("Reloading json resources from OpenSearch");
try {
- GetResponse response =
+ GetResponse<JsonData> response =
osClient.get(
- new GetRequest(
- "config", resource.getResourceFile()),
- RequestOptions.DEFAULT);
- resource.loadJSONResources(
- new ByteArrayInputStream(response.getSourceAsBytes()));
+ g ->
+ g.index("config")
+ .id(resource.getResourceFile()),
+ JsonData.class);
+ if (response.found() && response.source() != null) {
+ String json = response.source().toJson().toString();
+ resource.loadJSONResources(
+ new ByteArrayInputStream(
+ json.getBytes(StandardCharsets.UTF_8)));
+ }
} catch (Exception e) {
LOG.error("Can't load config from OpenSearch", e);
}
@@ -156,7 +165,7 @@ public class DelegateRefresher<T> {
}
if (osClient != null) {
try {
- osClient.close();
+ osClient._transport().close();
} catch (IOException e) {
LOG.error("Exception when closing OpenSearch client", e);
}
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java
index 180a1074..f7617205 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java
@@ -17,29 +17,22 @@
package org.apache.stormcrawler.opensearch;
-import com.google.common.base.Charsets;
import com.google.common.io.Resources;
import java.io.IOException;
import java.net.URL;
-import org.opensearch.OpenSearchException;
-import org.opensearch.action.support.master.AcknowledgedResponse;
-import org.opensearch.client.RequestOptions;
-import org.opensearch.client.RestHighLevelClient;
-import org.opensearch.client.indices.CreateIndexRequest;
-import org.opensearch.client.indices.CreateIndexResponse;
-import org.opensearch.client.indices.GetIndexRequest;
-import org.opensearch.client.indices.IndexTemplatesExistRequest;
-import org.opensearch.client.indices.PutIndexTemplateRequest;
-import org.opensearch.common.xcontent.XContentType;
+import java.nio.charset.StandardCharsets;
+import org.opensearch.client.opensearch.OpenSearchClient;
+import org.opensearch.client.opensearch.generic.Requests;
+import org.opensearch.client.opensearch.generic.Response;
+import org.opensearch.client.opensearch.indices.ExistsTemplateRequest;
import org.slf4j.Logger;
public class IndexCreation {
public static synchronized void checkOrCreateIndex(
- RestHighLevelClient client, String indexName, String boltType, Logger log)
+ OpenSearchClient client, String indexName, String boltType, Logger log)
throws IOException {
- final boolean indexExists =
- client.indices().exists(new GetIndexRequest(indexName), RequestOptions.DEFAULT);
+ final boolean indexExists = client.indices().exists(req -> req.index(indexName)).value();
log.info("Index '{}' exists? {}", indexName, indexExists);
// there's a possible check-then-update race condition
// createIndex intentionally catches and logs exceptions from OpenSearch
@@ -51,13 +44,12 @@ public class IndexCreation {
}
public static synchronized void checkOrCreateIndexTemplate(
- RestHighLevelClient client, String boltType, Logger log) throws IOException {
+ OpenSearchClient client, String boltType, Logger log) throws IOException {
final String templateName = boltType + "-template";
final boolean templateExists =
client.indices()
- .existsTemplate(
- new IndexTemplatesExistRequest(templateName),
- RequestOptions.DEFAULT);
+ .existsTemplate(ExistsTemplateRequest.of(r -> r.name(templateName)))
+ .value();
log.info("Template '{}' exists? {}", templateName, templateExists);
// there's a possible check-then-update race condition
// createTemplate intentionally catches and logs exceptions from OpenSearch
@@ -69,46 +61,54 @@ public class IndexCreation {
}
private static boolean createTemplate(
- RestHighLevelClient client, String templateName, String resourceName, Logger log) {
+ OpenSearchClient client, String templateName, String resourceName, Logger log) {
try {
- final PutIndexTemplateRequest createIndexRequest =
- new PutIndexTemplateRequest(templateName);
-
final URL mapping =
Thread.currentThread().getContextClassLoader().getResource(resourceName);
- final String jsonIndexConfiguration = Resources.toString(mapping, Charsets.UTF_8);
-
- createIndexRequest.source(jsonIndexConfiguration, XContentType.JSON);
-
- final AcknowledgedResponse createIndexResponse =
- client.indices().putTemplate(createIndexRequest, RequestOptions.DEFAULT);
- return createIndexResponse.isAcknowledged();
- } catch (IOException | OpenSearchException e) {
+ final String jsonIndexConfiguration =
+ Resources.toString(mapping, StandardCharsets.UTF_8);
+
+ try (Response response =
+ client.generic()
+ .execute(
+ Requests.builder()
+ .endpoint("/_template/" + templateName)
+ .method("PUT")
+ .json(jsonIndexConfiguration)
+ .build())) {
+ int statusCode = response.getStatus();
+ return statusCode == 200 || statusCode == 201;
+ }
+ } catch (Exception e) {
log.warn("template '{}' not created", templateName, e);
return false;
}
}
private static boolean createIndex(
- RestHighLevelClient client, String indexName, String resourceName, Logger log) {
+ OpenSearchClient client, String indexName, String resourceName, Logger log) {
try {
-
- final CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
-
final URL mapping =
Thread.currentThread().getContextClassLoader().getResource(resourceName);
- final String jsonIndexConfiguration = Resources.toString(mapping, Charsets.UTF_8);
-
- createIndexRequest.source(jsonIndexConfiguration, XContentType.JSON);
-
- final CreateIndexResponse createIndexResponse =
- client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
- return createIndexResponse.isAcknowledged();
- } catch (IOException | OpenSearchException e) {
+ final String jsonIndexConfiguration =
+ Resources.toString(mapping, StandardCharsets.UTF_8);
+
+ try (Response response =
+ client.generic()
+ .execute(
+ Requests.builder()
+ .endpoint("/" + indexName)
+ .method("PUT")
+ .json(jsonIndexConfiguration)
+ .build())) {
+ int statusCode = response.getStatus();
+ return statusCode == 200 || statusCode == 201;
+ }
+ } catch (Exception e) {
log.warn("index '{}' not created", indexName, e);
return false;
}
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
index f9a5dea1..eeb47362 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
@@ -17,8 +17,8 @@
package org.apache.stormcrawler.opensearch;
-import static org.opensearch.client.RestClientBuilder.DEFAULT_CONNECT_TIMEOUT_MILLIS;
-import static org.opensearch.client.RestClientBuilder.DEFAULT_SOCKET_TIMEOUT_MILLIS;
+import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_CONNECT_TIMEOUT_MILLIS;
+import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_RESPONSE_TIMEOUT_MILLIS;
import java.io.IOException;
import java.net.URI;
@@ -30,32 +30,29 @@ import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Pattern;
+import javax.net.ssl.SSLContext;
import org.apache.commons.lang3.StringUtils;
-import org.apache.http.HttpHost;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.Credentials;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
-import org.apache.http.conn.ssl.TrustAllStrategy;
-import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.hc.client5.http.auth.AuthScope;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.config.ConnectionConfig;
+import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
+import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
+import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
+import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.reactor.ssl.TlsDetails;
+import org.apache.hc.core5.ssl.SSLContextBuilder;
+import org.apache.hc.core5.util.Timeout;
import org.apache.stormcrawler.util.ConfUtils;
import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-import org.opensearch.action.DocWriteRequest;
-import org.opensearch.action.bulk.BulkProcessor;
-import org.opensearch.action.bulk.BulkRequest;
-import org.opensearch.action.bulk.BulkResponse;
-import org.opensearch.client.HttpAsyncResponseConsumerFactory;
-import org.opensearch.client.Node;
-import org.opensearch.client.RequestOptions;
-import org.opensearch.client.RestClient;
-import org.opensearch.client.RestClientBuilder;
-import org.opensearch.client.RestHighLevelClient;
-import org.opensearch.client.sniff.OpenSearchNodesSniffer;
-import org.opensearch.client.sniff.Sniffer;
-import org.opensearch.common.unit.TimeValue;
+import org.opensearch.client.json.jackson.JacksonJsonpMapper;
+import org.opensearch.client.opensearch.OpenSearchAsyncClient;
+import org.opensearch.client.opensearch.OpenSearchClient;
+import org.opensearch.client.opensearch.core.bulk.BulkOperation;
+import org.opensearch.client.transport.OpenSearchTransport;
+import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,24 +63,248 @@ public final class OpenSearchConnection {
private static final Logger LOG = LoggerFactory.getLogger(OpenSearchConnection.class);
- @NotNull private final RestHighLevelClient client;
+ @NotNull private final OpenSearchClient client;
- @NotNull private final BulkProcessor processor;
+ @NotNull private final AsyncBulkProcessor processor;
- @Nullable private final Sniffer sniffer;
+ @NotNull private final OpenSearchTransport transport;
private OpenSearchConnection(
- @NotNull RestHighLevelClient c, @NotNull BulkProcessor p, @Nullable Sniffer s) {
- processor = p;
+ @NotNull OpenSearchClient c,
+ @NotNull AsyncBulkProcessor p,
+ @NotNull OpenSearchTransport t) {
client = c;
- sniffer = s;
+ processor = p;
+ transport = t;
}
- public RestHighLevelClient getClient() {
+ public OpenSearchClient getClient() {
return client;
}
- public static RestHighLevelClient getClient(Map<String, Object> stormConf, String boltType) {
+ /**
+ * Creates a standalone {@link OpenSearchClient}. Used by classes that need a client without a
+ * bulk processor (e.g. spouts, filters). Callers are responsible for closing the returned
+ * client's transport via {@code client._transport().close()}.
+ */
+ public static OpenSearchClient getClient(Map<String, Object> stormConf, String boltType) {
+ return buildClientResources(stormConf, boltType).client();
+ }
+
+ /** Adds a single bulk operation to the internal processor. */
+ public void addToProcessor(final BulkOperation operation) {
+ processor.add(operation);
+ }
+
+ /**
+ * Creates a connection with a default (no-op) listener. The values for bolt type are
+ * [indexer,status,metrics].
+ */
+ public static OpenSearchConnection getConnection(
+ Map<String, Object> stormConf, String boltType) {
+ AsyncBulkProcessor.Listener listener =
+ new AsyncBulkProcessor.Listener() {
+ @Override
+ public void afterBulk(
+ long arg0,
+ org.opensearch.client.opensearch.core.BulkRequest arg1,
+ org.opensearch.client.opensearch.core.BulkResponse arg2) {}
+
+ @Override
+ public void afterBulk(
+ long arg0,
+ org.opensearch.client.opensearch.core.BulkRequest arg1,
+ Throwable arg2) {}
+
+ @Override
+ public void beforeBulk(
+ long arg0, org.opensearch.client.opensearch.core.BulkRequest arg1) {}
+ };
+ return getConnection(stormConf, boltType, listener);
+ }
+
+ public static OpenSearchConnection getConnection(
+ Map<String, Object> stormConf, String boltType, AsyncBulkProcessor.Listener listener) {
+
+ final String dottedType = boltType + ".";
+
+ warnOnRemovedKeys(stormConf, dottedType);
+
+ ClientResources cr = buildClientResources(stormConf, boltType);
+
+ final String flushIntervalString =
+ ConfUtils.getString(
+ stormConf, Constants.PARAMPREFIX, dottedType, "flushInterval", "5s");
+
+ final long flushIntervalMillis = parseTimeValueToMillis(flushIntervalString, 5000);
+
+ final int bulkActions =
+ ConfUtils.getInt(stormConf, Constants.PARAMPREFIX, dottedType, "bulkActions", 50);
+
+ final int concurrentRequests =
+ ConfUtils.getInt(
+ stormConf, Constants.PARAMPREFIX, dottedType, "concurrentRequests", 1);
+
+ AsyncBulkProcessor bulkProcessor = null;
+ try {
+ bulkProcessor =
+ new AsyncBulkProcessor.Builder(
+ new OpenSearchAsyncClient(cr.transport()), listener)
+ .setBulkActions(bulkActions)
+ .setFlushIntervalMillis(flushIntervalMillis)
+ .setConcurrentRequests(concurrentRequests)
+ .build();
+
+ return new OpenSearchConnection(cr.client(), bulkProcessor, cr.transport());
+ } catch (Exception e) {
+ if (bulkProcessor != null) {
+ try {
+ bulkProcessor.close();
+ } catch (Exception suppressed) {
+ e.addSuppressed(suppressed);
+ }
+ }
+ try {
+ cr.transport().close();
+ } catch (IOException suppressed) {
+ e.addSuppressed(suppressed);
+ }
+ throw e;
+ }
+ }
+
+ private final AtomicBoolean isClosed = new AtomicBoolean(false);
+
+ public void close() {
+
+ if (!isClosed.compareAndSet(false, true)) {
+ LOG.warn("Tried to close an already closed connection!");
+ return;
+ }
+
+ LOG.debug("Start closing the OpenSearch connection");
+
+ // First, close the BulkProcessor ensuring pending actions are flushed
+ try {
+ boolean success = processor.awaitClose(60, TimeUnit.SECONDS);
+ if (!success) {
+ throw new RuntimeException(
+ "Failed to flush pending actions when closing BulkProcessor");
+ }
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ // Now close the transport (also shuts down the underlying HTTP client)
+ try {
+ transport.close();
+ } catch (IOException e) {
+ LOG.trace("Transport threw IO exception on close.");
+ }
+ }
+
+ /**
+ * Extracts the document ID from a {@link BulkOperation} regardless of its type (index, create,
+ * delete, update).
+ */
+ public static String getBulkOperationId(BulkOperation op) {
+ if (op.isIndex()) {
+ return op.index().id();
+ }
+ if (op.isCreate()) {
+ return op.create().id();
+ }
+ if (op.isDelete()) {
+ return op.delete().id();
+ }
+ if (op.isUpdate()) {
+ return op.update().id();
+ }
+ return null;
+ }
+
+ /**
+ * Returns the scopes the Basic credentials are registered for: the scheme, host and port of
+ * each configured address. A request to any other address does not receive them.
+ */
+ static List<AuthScope> credentialScopes(List<HttpHost> hosts) {
+ final Set<AuthScope> scopes = new LinkedHashSet<>();
+ for (HttpHost host : hosts) {
+ scopes.add(new AuthScope(host));
+ }
+ return new ArrayList<>(scopes);
+ }
+
+ /**
+ * Returns the addresses which use plain http and are not a loopback address, i.e. those the
+ * Basic credentials would be sent to in the clear over the network.
+ */
+ static List<HttpHost> plainHttpHosts(List<HttpHost> hosts) {
+ final List<HttpHost> plain = new ArrayList<>();
+ for (HttpHost host : hosts) {
+ if ("http".equalsIgnoreCase(host.getSchemeName()) && !isLoopback(host.getHostName())) {
+ plain.add(host);
+ }
+ }
+ return plain;
+ }
+
+ private static final Pattern IPV4_LOOPBACK = Pattern.compile("127(\\.\\d{1,3}){3}");
+
+ /** Whether the host is localhost or a loopback IP literal. No name is resolved. */
+ static boolean isLoopback(String hostname) {
+ String host = StringUtils.strip(hostname.toLowerCase(Locale.ROOT), "[]");
+ return host.equals("localhost")
+ || IPV4_LOOPBACK.matcher(host).matches()
+ || host.equals("::1")
+ || host.equals("0:0:0:0:0:0:0:1");
+ }
+
+ private static void warnAboutPlainHttp(String boltType, List<HttpHost> hosts) {
+ final List<HttpHost> plain = plainHttpHosts(hosts);
+ if (!plain.isEmpty()) {
+ LOG.warn(
+ "OpenSearch credentials are configured for {} but the addresses {} use plain "
+ + "http, the credentials are sent unencrypted. Give the addresses an "
+ + "https:// scheme.",
+ boltType,
+ plain);
+ }
+ }
+
+ // internal helpers
+ private record ClientResources(OpenSearchClient client, OpenSearchTransport transport) {}
+
+ /**
+ * Logs a WARN for legacy configuration keys that are no longer honoured by this module, so that
+ * users migrating from {@code external/opensearch} notice silently-dropped tuning. See the
+ * module README for the full list of differences.
+ */
+ private static void warnOnRemovedKeys(Map<String, Object> stormConf, String dottedType) {
+ final String responseBufferKey = Constants.PARAMPREFIX + dottedType + "responseBufferSize";
+ if (stormConf.containsKey(responseBufferKey)) {
+ LOG.warn(
+ "Configuration key '{}' is set but no longer supported by the opensearch-java module. "
+ + "The HC5-based async transport does not expose an equivalent per-request "
+ + "heap-buffer override. The setting is ignored — remove it from your "
+ + "configuration. See external/opensearch-java/README.md for details.",
+ responseBufferKey);
+ }
+
+ final String sniffKey = Constants.PARAMPREFIX + dottedType + "sniff";
+ if (stormConf.containsKey(sniffKey)) {
+ LOG.warn(
+ "Configuration key '{}' is set but no longer supported by the opensearch-java module. "
+ + "The OpenSearch Java Client 3.x does not ship a Sniffer equivalent, so "
+ + "automatic node discovery is not available. Keep the 'addresses' list up to "
+ + "date manually or put a load balancer in front of the cluster. "
+ + "See external/opensearch-java/README.md for details.",
+ sniffKey);
+ }
+ }
+
+ private static ClientResources buildClientResources(
+ Map<String, Object> stormConf, String boltType) {
final String dottedType = boltType + ".";
@@ -119,10 +340,15 @@ public final class OpenSearchConnection {
if (uri.getScheme() != null) {
scheme = uri.getScheme();
}
- hosts.add(new HttpHost(uri.getHost(), port, scheme));
+ // HC5: constructor is (scheme, hostname, port) — not (hostname, port, scheme)
+ hosts.add(new HttpHost(scheme, uri.getHost(), port));
}
- final RestClientBuilder builder = RestClient.builder(hosts.toArray(new HttpHost[0]));
+ LOG.info(
+ "OpenSearch {} transport configured with {} host(s): {}",
+ boltType,
+ hosts.size(),
+ hosts);
// authentication via user / password
final String user =
@@ -144,9 +370,40 @@ public final class OpenSearchConnection {
ConfUtils.getBoolean(
stormConf, Constants.PARAMPREFIX, "", "disable.tls.validation", false);
- final boolean needsUser = hasCredentials(user, password);
+ final boolean needsUser = StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password);
final boolean needsProxy = StringUtils.isNotBlank(proxyhost) && proxyport != -1;
+ // Defaults from ApacheHttpClient5TransportBuilder (same as the former RestClientBuilder)
+ final int connectTimeout =
+ ConfUtils.getInt(
+ stormConf,
+ Constants.PARAMPREFIX,
+ dottedType,
+ "connect.timeout",
+ DEFAULT_CONNECT_TIMEOUT_MILLIS);
+ final int socketTimeout =
+ ConfUtils.getInt(
+ stormConf,
+ Constants.PARAMPREFIX,
+ dottedType,
+ "socket.timeout",
+ DEFAULT_RESPONSE_TIMEOUT_MILLIS);
+
+ final boolean compression =
+ ConfUtils.getBoolean(
+ stormConf, Constants.PARAMPREFIX, dottedType, "compression", false);
+
+ final ApacheHttpClient5TransportBuilder builder =
+ ApacheHttpClient5TransportBuilder.builder(hosts.toArray(new HttpHost[0]))
+ .setMapper(new JacksonJsonpMapper());
+
+ // Timeouts via ConnectionConfig on the builder's internal connection manager
+ builder.setConnectionConfigCallback(
+ connConfigBuilder ->
+ connConfigBuilder
+ .setConnectTimeout(Timeout.ofMilliseconds(connectTimeout))
+ .setSocketTimeout(Timeout.ofMilliseconds(socketTimeout)));
+
if (needsUser) {
warnAboutPlainHttp(boltType, hosts);
}
@@ -157,27 +414,61 @@ public final class OpenSearchConnection {
boltType);
}
+ // Auth, proxy, and/or trust-all SSL via HttpClient customisation
if (needsUser || needsProxy || disableTlsValidation) {
builder.setHttpClientConfigCallback(
httpClientBuilder -> {
+ // hc.client5 auth: password is char[], AuthScope(host, port)
if (needsUser) {
- httpClientBuilder.setDefaultCredentialsProvider(
- new OriginCredentialsProvider(
- hosts,
- new UsernamePasswordCredentials(user, password)));
+ final BasicCredentialsProvider credentialsProvider =
+ new BasicCredentialsProvider();
+ final UsernamePasswordCredentials credentials =
+ new UsernamePasswordCredentials(user, password.toCharArray());
+ for (AuthScope scope : credentialScopes(hosts)) {
+ credentialsProvider.setCredentials(scope, credentials);
+ }
+ httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
}
+ // hc.client5 proxy: HttpHost(scheme, host, port)
if (needsProxy) {
httpClientBuilder.setProxy(
- new HttpHost(proxyhost, proxyport, proxyscheme));
+ new HttpHost(proxyscheme, proxyhost, proxyport));
}
-
+ // Custom connection manager overrides the builder's internal one,
+ // so timeouts and TlsDetailsFactory must be replicated here
if (disableTlsValidation) {
try {
- final SSLContextBuilder sslContext = new SSLContextBuilder();
- sslContext.loadTrustMaterial(null, new TrustAllStrategy());
- httpClientBuilder.setSSLContext(sslContext.build());
- httpClientBuilder.setSSLHostnameVerifier(
- NoopHostnameVerifier.INSTANCE);
+ final SSLContext sslContext =
+ SSLContextBuilder.create()
+ .loadTrustMaterial((chain, authType) -> true)
+ .build();
+ httpClientBuilder.setConnectionManager(
+ PoolingAsyncClientConnectionManagerBuilder.create()
+ .setTlsStrategy(
+ ClientTlsStrategyBuilder.create()
+ .setSslContext(sslContext)
+ .setHostnameVerifier(
+ NoopHostnameVerifier
+ .INSTANCE)
+ // HTTP/2 ALPN negotiation
+ .setTlsDetailsFactory(
+ sslEngine ->
+ new TlsDetails(
+ sslEngine
+ .getSession(),
+ sslEngine
+ .getApplicationProtocol()))
+ .build())
+ .setDefaultConnectionConfig(
+ ConnectionConfig.custom()
+ .setConnectTimeout(
+ Timeout.ofMilliseconds(
+ connectTimeout))
+ .setSocketTimeout(
+ Timeout.ofMilliseconds(
+ socketTimeout))
+ .build())
+ .build());
} catch (Exception e) {
throw new RuntimeException("Failed to disable TLS validation", e);
}
@@ -186,324 +477,40 @@ public final class OpenSearchConnection {
});
}
- final int connectTimeout =
- ConfUtils.getInt(
- stormConf,
- Constants.PARAMPREFIX,
- dottedType,
- "connect.timeout",
- DEFAULT_CONNECT_TIMEOUT_MILLIS);
- final int socketTimeout =
- ConfUtils.getInt(
- stormConf,
- Constants.PARAMPREFIX,
- dottedType,
- "socket.timeout",
- DEFAULT_SOCKET_TIMEOUT_MILLIS);
- // timeout until connection is established
- builder.setRequestConfigCallback(
- requestConfigBuilder ->
- requestConfigBuilder
- .setConnectTimeout(connectTimeout)
- // Timeout when waiting for data
- .setSocketTimeout(socketTimeout));
-
- // TODO check if this has gone somewhere else
- // int maxRetryTimeout = ConfUtils.getInt(stormConf, Constants.PARAMPREFIX +
- // boltType +
- // ".max.retry.timeout",
- // DEFAULT_MAX_RETRY_TIMEOUT_MILLIS);
- // builder.setMaxRetryTimeoutMillis(maxRetryTimeout);
-
- // TODO configure headers etc...
- // Map<String, String> configSettings = (Map) stormConf
- // .get(Constants.PARAMPREFIX + boltType + ".settings");
- // if (configSettings != null) {
- // configSettings.forEach((k, v) -> settings.put(k, v));
- // }
-
- // use node selector only to log nodes listed in the config
- // and/or discovered through sniffing
- builder.setNodeSelector(
- nodes -> {
- for (Node node : nodes) {
- LOG.debug(
- "Connected to OpenSearch node {} [{}] for {}",
- node.getName(),
- node.getHost(),
- boltType);
- }
- });
-
- final boolean compression =
- ConfUtils.getBoolean(
- stormConf, Constants.PARAMPREFIX, dottedType, "compression", false);
-
+ // Compression: first-class builder method, not a request interceptor
builder.setCompressionEnabled(compression);
- return new RestHighLevelClient(builder);
- }
+ final OpenSearchTransport transport = builder.build();
+ final OpenSearchClient openSearchClient = new OpenSearchClient(transport);
- /** Basic authentication is only set up when both the user and the password are given. */
- static boolean hasCredentials(String user, String password) {
- return StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password);
+ return new ClientResources(openSearchClient, transport);
}
/**
- * Gives the Basic credentials only to requests for one of the configured addresses, matched on
- * scheme, host and port. A node reached under any other address, such as one found by the
- * sniffer under the address it publishes, does not receive them.
+ * Parses a time value string (e.g. "5s", "500ms", "1m") into milliseconds.
+ *
+ * @param value the string to parse
+ * @param defaultMillis the default if parsing fails
+ * @return milliseconds
*/
- static final class OriginCredentialsProvider implements CredentialsProvider {
-
- private final Set<String> origins = new LinkedHashSet<>();
- private final Credentials credentials;
-
- OriginCredentialsProvider(List<HttpHost> hosts, Credentials credentials) {
- for (HttpHost host : hosts) {
- origins.add(origin(host));
- }
- this.credentials = credentials;
- }
-
- private static String origin(HttpHost host) {
- final String scheme = host.getSchemeName().toLowerCase(Locale.ROOT);
- int port = host.getPort();
- if (port < 0) {
- port = "https".equals(scheme) ? 443 : 80;
- }
- return scheme + "://" + host.getHostName().toLowerCase(Locale.ROOT) + ":" + port;
- }
-
- @Override
- public Credentials getCredentials(AuthScope scope) {
- final HttpHost origin = scope.getOrigin();
- if (origin != null && origins.contains(origin(origin))) {
- return credentials;
- }
- return null;
+ static long parseTimeValueToMillis(String value, long defaultMillis) {
+ if (value == null || value.isBlank()) {
+ return defaultMillis;
}
-
- @Override
- public void setCredentials(AuthScope scope, Credentials credentials) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void clear() {}
- }
-
- /**
- * Returns the addresses which use plain http and are not a loopback address, i.e. those the
- * Basic credentials would be sent to in the clear over the network.
- */
- static List<HttpHost> plainHttpHosts(List<HttpHost> hosts) {
- final List<HttpHost> plain = new ArrayList<>();
- for (HttpHost host : hosts) {
- if ("http".equalsIgnoreCase(host.getSchemeName()) && !isLoopback(host.getHostName())) {
- plain.add(host);
- }
- }
- return plain;
- }
-
- /**
- * Returns the scheme under which the sniffer registers the nodes it finds. The nodes publish no
- * scheme, so https is used as soon as one configured address uses it; otherwise sniffing would
- * move the requests to plain http after the first round.
- */
- static OpenSearchNodesSniffer.Scheme sniffScheme(List<HttpHost> hosts) {
- for (HttpHost host : hosts) {
- if ("https".equalsIgnoreCase(host.getSchemeName())) {
- return OpenSearchNodesSniffer.Scheme.HTTPS;
- }
- }
- return OpenSearchNodesSniffer.Scheme.HTTP;
- }
-
- private static final Pattern IPV4_LOOPBACK = Pattern.compile("127(\\.\\d{1,3}){3}");
-
- /** Whether the host is localhost or a loopback IP literal. No name is resolved. */
- static boolean isLoopback(String hostname) {
- String host = StringUtils.strip(hostname.toLowerCase(Locale.ROOT), "[]");
- return host.equals("localhost")
- || IPV4_LOOPBACK.matcher(host).matches()
- || host.equals("::1")
- || host.equals("0:0:0:0:0:0:0:1");
- }
-
- private static void warnAboutPlainHttp(String boltType, List<HttpHost> hosts) {
- final List<HttpHost> plain = plainHttpHosts(hosts);
- if (!plain.isEmpty()) {
- LOG.warn(
- "OpenSearch credentials are configured for {} but the addresses {} use plain "
- + "http, the credentials are sent unencrypted. Give the addresses an "
- + "https:// scheme.",
- boltType,
- plain);
- }
- }
-
- public void addToProcessor(final DocWriteRequest<?> request) {
- processor.add(request);
- }
-
- /**
- * Creates a connection with a default listener. The values for bolt type are
- * [indexer,status,metrics]
- */
- public static OpenSearchConnection getConnection(
- Map<String, Object> stormConf, String boltType) {
- BulkProcessor.Listener listener =
- new BulkProcessor.Listener() {
- @Override
- public void afterBulk(long arg0, BulkRequest arg1, BulkResponse arg2) {}
-
- @Override
- public void afterBulk(long arg0, BulkRequest arg1, Throwable arg2) {}
-
- @Override
- public void beforeBulk(long arg0, BulkRequest arg1) {}
- };
- return getConnection(stormConf, boltType, listener);
- }
-
- public static OpenSearchConnection getConnection(
- Map<String, Object> stormConf, String boltType, BulkProcessor.Listener listener) {
-
- final RestHighLevelClient client = getClient(stormConf, boltType);
-
- final String dottedType = boltType + ".";
-
- final String flushIntervalString =
- ConfUtils.getString(
- stormConf, Constants.PARAMPREFIX, dottedType, "flushInterval", "5s");
-
- final TimeValue flushInterval =
- TimeValue.parseTimeValue(
- flushIntervalString, TimeValue.timeValueSeconds(5), "flushInterval");
-
- final int bulkActions =
- ConfUtils.getInt(stormConf, Constants.PARAMPREFIX, dottedType, "bulkActions", 50);
-
- final int concurrentRequests =
- ConfUtils.getInt(
- stormConf, Constants.PARAMPREFIX, dottedType, "concurrentRequests", 1);
-
- final RequestOptions requestOptions = RequestOptions.DEFAULT;
- final RequestOptions.Builder requestOptionsBuilder = requestOptions.toBuilder();
- final int bufferSize =
- ConfUtils.getInt(
- stormConf, Constants.PARAMPREFIX, dottedType, "responseBufferSize", 100);
-
- requestOptionsBuilder.setHttpAsyncResponseConsumerFactory(
- new HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory(
- bufferSize * 1024 * 1024));
-
- BulkProcessor bulkProcessor = null;
- Sniffer sniffer = null;
- try {
- bulkProcessor =
- BulkProcessor.builder(
- (request, bulkListener) ->
- client.bulkAsync(
- request,
- requestOptionsBuilder.build(),
- bulkListener),
- listener)
- .setFlushInterval(flushInterval)
- .setBulkActions(bulkActions)
- .setConcurrentRequests(concurrentRequests)
- .build();
-
- boolean sniff =
- ConfUtils.getBoolean(
- stormConf, Constants.PARAMPREFIX, dottedType, "sniff", true);
- if (sniff) {
- if (hasCredentials(
- ConfUtils.getString(stormConf, Constants.PARAMPREFIX, dottedType, "user"),
- ConfUtils.getString(
- stormConf, Constants.PARAMPREFIX, dottedType, "password"))) {
- LOG.warn(
- "Sniffing is enabled for {} and OpenSearch credentials are configured. "
- + "The credentials are only sent to the configured addresses: "
- + "requests to a node the sniffer finds under another host or "
- + "port are sent without them and fail if the cluster requires "
- + "authentication. List every node in opensearch.{}.addresses "
- + "or set opensearch.{}.sniff: false.",
- boltType,
- boltType,
- boltType);
- }
- final RestClient lowLevelClient = client.getLowLevelClient();
- final List<HttpHost> configured = new ArrayList<>();
- for (Node node : lowLevelClient.getNodes()) {
- configured.add(node.getHost());
- }
- sniffer =
- Sniffer.builder(lowLevelClient)
- .setNodesSniffer(
- new OpenSearchNodesSniffer(
- lowLevelClient,
- OpenSearchNodesSniffer
- .DEFAULT_SNIFF_REQUEST_TIMEOUT,
- sniffScheme(configured)))
- .build();
- }
-
- return new OpenSearchConnection(client, bulkProcessor, sniffer);
- } catch (Exception e) {
- if (bulkProcessor != null) {
- try {
- bulkProcessor.close();
- } catch (Exception suppressed) {
- e.addSuppressed(suppressed);
- }
- }
- try {
- client.close();
- } catch (IOException suppressed) {
- e.addSuppressed(suppressed);
- }
- throw e;
- }
- }
-
- private boolean isClosed = false;
-
- public void close() {
-
- if (isClosed) {
- LOG.warn("Tried to close an already closed connection!");
- return;
- }
-
- // Maybe some kind of identifier?
- LOG.debug("Start closing the OpenSearch connection");
-
- // First, close the BulkProcessor ensuring pending actions are flushed
+ value = value.trim();
try {
- boolean success = processor.awaitClose(60, TimeUnit.SECONDS);
- if (!success) {
- throw new RuntimeException(
- "Failed to flush pending actions when closing BulkProcessor");
+ if (value.endsWith("ms")) {
+ return Long.parseLong(value.substring(0, value.length() - 2));
+ } else if (value.endsWith("s")) {
+ return Long.parseLong(value.substring(0, value.length() - 1)) * 1000;
+ } else if (value.endsWith("m")) {
+ return Long.parseLong(value.substring(0, value.length() - 1)) * 60000;
+ } else {
+ return Long.parseLong(value);
}
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
-
- if (sniffer != null) {
- sniffer.close();
+ } catch (NumberFormatException e) {
+ LOG.warn("Could not parse time value '{}', using default {}ms", value, defaultMillis);
+ return defaultMillis;
}
-
- // Now close the actual client
- try {
- client.close();
- } catch (IOException e) {
- // ignore silently
- LOG.trace("Client threw IO exception.");
- }
-
- isClosed = true;
}
}
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java
index bdb9a8dc..0e8574fb 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java
@@ -21,23 +21,22 @@ import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.RemovalCause;
import com.github.benmanes.caffeine.cache.Ticker;
-import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.apache.storm.tuple.Tuple;
import org.apache.stormcrawler.metrics.ScopedCounter;
import org.jetbrains.annotations.Nullable;
-import org.opensearch.action.DocWriteRequest;
-import org.opensearch.action.bulk.BulkItemResponse;
-import org.opensearch.action.bulk.BulkRequest;
-import org.opensearch.action.bulk.BulkResponse;
-import org.opensearch.core.rest.RestStatus;
+// opensearch-java: uses typed BulkRequest/BulkResponse, not legacy REST equivalents
+import org.opensearch.client.opensearch.core.BulkRequest;
+import org.opensearch.client.opensearch.core.BulkResponse;
import org.slf4j.Logger;
/**
@@ -54,8 +53,7 @@ public class WaitAckCache {
}
private final Cache<String, List<Tuple>> cache;
- private final java.util.concurrent.locks.ReentrantLock lock =
- new java.util.concurrent.locks.ReentrantLock(true);
+ private final ReentrantLock lock = new ReentrantLock(true);
private final Logger log;
private final Consumer<Tuple> onEviction;
@@ -64,7 +62,10 @@ public class WaitAckCache {
this(Caffeine.newBuilder().expireAfterWrite(60, TimeUnit.SECONDS), log, onEviction);
}
- /** Creates a cache from a Caffeine spec string (e.g. "expireAfterWrite=300s"). */
+ /**
+ * Creates a cache using a {@link Caffeine} spec string (e.g. {@code "expireAfterWrite=300s"}),
+ * typically driven by {@code topology.message.timeout.secs}.
+ */
public WaitAckCache(String cacheSpec, Logger log, Consumer<Tuple> onEviction) {
this(Caffeine.from(cacheSpec), log, onEviction);
}
@@ -98,6 +99,7 @@ public class WaitAckCache {
.build();
}
+ /** Returns the approximate number of entries in this cache. */
public long estimatedSize() {
return cache.estimatedSize();
}
@@ -176,21 +178,27 @@ public class WaitAckCache {
@Nullable ScopedCounter conflictCounter,
TupleAction action) {
+ // opensearch-java: items() returns List<BulkResponseItem>; status() returns int
var idsToBulkItems =
- Arrays.stream(response.getItems())
+ response.items().stream()
.map(
bir -> {
- BulkItemResponse.Failure f = bir.getFailure();
+ var error = bir.error();
boolean failed = false;
- if (f != null) {
- if (f.getStatus().equals(RestStatus.CONFLICT)) {
+ if (error != null) {
+ // opensearch-java: int status code, not RestStatus enum
+ if (bir.status() == 409) {
if (conflictCounter != null) {
conflictCounter.scope("doc_conflicts").incrBy(1);
}
- log.debug("Doc conflict ID {}", bir.getId());
+ log.debug("Doc conflict ID {}", bir.id());
} else {
log.error(
- "Bulk item failure ID {}: {}", bir.getId(), f);
+ "Bulk item failure ID {}: {}",
+ bir.id(),
+ error.reason() != null
+ ? error.reason()
+ : error.type());
failed = true;
}
}
@@ -198,7 +206,9 @@ public class WaitAckCache {
})
.collect(
// https://github.com/apache/stormcrawler/issues/832
- Collectors.groupingBy(b -> b.id, Collectors.toUnmodifiableList()));
+ Collectors.groupingBy(
+ BulkItemResponseToFailedFlag::id,
+ Collectors.toUnmodifiableList()));
Map<String, List<Tuple>> presentTuples;
long estimatedSize;
@@ -230,7 +240,7 @@ public class WaitAckCache {
if (tuples != null) {
log.debug("Found {} tuple(s) for ID {}", tuples.size(), id);
for (Tuple t : tuples) {
- if (selected.failed) {
+ if (selected.failed()) {
failureCount++;
} else {
ackCount++;
@@ -257,15 +267,22 @@ public class WaitAckCache {
}
}
- /** Processes a failed bulk request by failing all associated tuples. */
+ /**
+ * Processes a failed bulk request by failing all associated tuples.
+ *
+ * @param failAction callback applied to each tuple that must be failed
+ */
public void processFailedBulk(
BulkRequest request, long executionId, Throwable failure, Consumer<Tuple> failAction) {
log.error("Exception with bulk {} - failing the whole lot ", executionId, failure);
+ // opensearch-java: operations() + getBulkOperationId replaces
+ // legacy requests() + DocWriteRequest::id
final var failedIds =
- request.requests().stream()
- .map(DocWriteRequest::id)
+ request.operations().stream()
+ .map(OpenSearchConnection::getBulkOperationId)
+ .filter(Objects::nonNull)
.collect(Collectors.toUnmodifiableSet());
Map<String, List<Tuple>> failedTupleLists;
@@ -306,7 +323,7 @@ public class WaitAckCache {
BulkItemResponseToFailedFlag best = items.get(0);
int failedCount = 0;
for (var item : items) {
- if (item.failed) {
+ if (item.failed()) {
failedCount++;
} else {
best = item;
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
index 16b3a005..f77662ea 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
@@ -28,13 +28,13 @@ import org.apache.storm.tuple.Tuple;
import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
import org.apache.stormcrawler.metrics.CrawlerMetrics;
+import org.apache.stormcrawler.opensearch.AsyncBulkProcessor;
import org.apache.stormcrawler.opensearch.OpenSearchConnection;
import org.apache.stormcrawler.opensearch.WaitAckCache;
import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.action.bulk.BulkProcessor.Listener;
-import org.opensearch.action.bulk.BulkRequest;
-import org.opensearch.action.bulk.BulkResponse;
-import org.opensearch.action.delete.DeleteRequest;
+import org.opensearch.client.opensearch.core.BulkRequest;
+import org.opensearch.client.opensearch.core.BulkResponse;
+import org.opensearch.client.opensearch.core.bulk.BulkOperation;
import org.slf4j.LoggerFactory;
/**
@@ -43,7 +43,7 @@ import org.slf4j.LoggerFactory;
* will also try to delete documents even though they were never indexed and it currently won't
* delete documents which were indexed under the canonical URL.
*/
-public class DeletionBolt extends BaseRichBolt implements Listener {
+public class DeletionBolt extends BaseRichBolt implements AsyncBulkProcessor.Listener {
static final org.slf4j.Logger LOG =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -81,7 +81,7 @@ public class DeletionBolt extends BaseRichBolt implements Listener {
try {
connection = OpenSearchConnection.getConnection(conf, BOLT_TYPE, this);
} catch (Exception e1) {
- LOG.error("Can't connect to opensearch", e1);
+ LOG.error("Can't connect to OpenSearch", e1);
throw new RuntimeException(e1);
}
@@ -106,10 +106,12 @@ public class DeletionBolt extends BaseRichBolt implements Listener {
// used
final String docID = getDocumentID(metadata, url);
- DeleteRequest dr = new DeleteRequest(getIndexName(metadata), docID);
- connection.addToProcessor(dr);
+ final String targetIndex = getIndexName(metadata);
+ BulkOperation op = BulkOperation.of(b -> b.delete(d -> d.index(targetIndex).id(docID)));
waitAck.addTuple(docID, tuple);
+
+ connection.addToProcessor(op);
}
@Override
@@ -155,9 +157,9 @@ public class DeletionBolt extends BaseRichBolt implements Listener {
waitAck.processBulkResponse(
response,
executionId,
- null, // no conflict counter — deletion conflicts are expected and only logged
+ null,
(id, t, selected) -> {
- if (!selected.failed) {
+ if (!selected.failed()) {
_collector.ack(t);
} else {
String url = (String) t.getValueByField("url");
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
index 6191f2c3..f7a44185 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java
@@ -18,9 +18,10 @@
package org.apache.stormcrawler.opensearch.bolt;
import static org.apache.stormcrawler.Constants.StatusStreamName;
-import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.commons.lang3.StringUtils;
@@ -34,17 +35,15 @@ import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
import org.apache.stormcrawler.metrics.CrawlerMetrics;
import org.apache.stormcrawler.metrics.ScopedCounter;
import org.apache.stormcrawler.metrics.ScopedReducedMetric;
+import org.apache.stormcrawler.opensearch.AsyncBulkProcessor;
import org.apache.stormcrawler.opensearch.IndexCreation;
import org.apache.stormcrawler.opensearch.OpenSearchConnection;
import org.apache.stormcrawler.opensearch.WaitAckCache;
import org.apache.stormcrawler.persistence.Status;
import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.action.bulk.BulkProcessor;
-import org.opensearch.action.bulk.BulkRequest;
-import org.opensearch.action.bulk.BulkResponse;
-import org.opensearch.action.index.IndexRequest;
-import org.opensearch.core.rest.RestStatus;
-import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.client.opensearch.core.BulkRequest;
+import org.opensearch.client.opensearch.core.BulkResponse;
+import org.opensearch.client.opensearch.core.bulk.BulkOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,7 +51,7 @@ import org.slf4j.LoggerFactory;
* Sends documents to opensearch. Indexes all the fields from the tuples or a Map
* <String,Object> from a named field.
*/
-public class IndexerBolt extends AbstractIndexerBolt implements BulkProcessor.Listener {
+public class IndexerBolt extends AbstractIndexerBolt implements AsyncBulkProcessor.Listener {
private static final Logger LOG = LoggerFactory.getLogger(IndexerBolt.class);
@@ -105,7 +104,7 @@ public class IndexerBolt extends AbstractIndexerBolt implements BulkProcessor.Li
try {
connection = OpenSearchConnection.getConnection(conf, OSBoltType, this);
} catch (Exception e1) {
- LOG.error("Can't connect to opensearch", e1);
+ LOG.error("Can't connect to OpenSearch", e1);
throw new RuntimeException(e1);
}
@@ -159,19 +158,19 @@ public class IndexerBolt extends AbstractIndexerBolt implements BulkProcessor.Li
final String docID = getDocumentID(metadata, normalisedurl);
try {
- final XContentBuilder builder = jsonBuilder().startObject();
+ final Map<String, Object> source = new HashMap<>();
// display text of the document?
if (StringUtils.isNotBlank(fieldNameForText())) {
final String text = trimText(tuple.getStringByField("text"));
if (!ignoreEmptyFields() || StringUtils.isNotBlank(text)) {
- builder.field(fieldNameForText(), trimText(text));
+ source.put(fieldNameForText(), trimText(text));
}
}
// send URL as field?
if (StringUtils.isNotBlank(fieldNameForURL())) {
- builder.field(fieldNameForURL(), normalisedurl);
+ source.put(fieldNameForURL(), normalisedurl);
}
// which metadata to display?
@@ -181,35 +180,54 @@ public class IndexerBolt extends AbstractIndexerBolt implements BulkProcessor.Li
if (entry.getValue().length == 1) {
final String value = entry.getValue()[0];
if (!ignoreEmptyFields() || StringUtils.isNotBlank(value)) {
- builder.field(entry.getKey(), value);
+ source.put(entry.getKey(), value);
}
} else if (entry.getValue().length > 1) {
- builder.array(entry.getKey(), entry.getValue());
+ source.put(entry.getKey(), Arrays.asList(entry.getValue()));
}
}
- builder.endObject();
-
- final IndexRequest indexRequest =
- new IndexRequest(getIndexName(metadata))
- .source(builder)
- .id(docID)
- .create(create);
-
- if (pipeline != null) {
- indexRequest.setPipeline(pipeline);
+ final String targetIndex = getIndexName(metadata);
+ final BulkOperation op;
+ if (create) {
+ op =
+ BulkOperation.of(
+ b ->
+ b.create(
+ c -> {
+ c.index(targetIndex).id(docID).document(source);
+ if (pipeline != null) {
+ c.pipeline(pipeline);
+ }
+ return c;
+ }));
+ } else {
+ op =
+ BulkOperation.of(
+ b ->
+ b.index(
+ idx -> {
+ idx.index(targetIndex)
+ .id(docID)
+ .document(source);
+ if (pipeline != null) {
+ idx.pipeline(pipeline);
+ }
+ return idx;
+ }));
}
- connection.addToProcessor(indexRequest);
+ waitAck.addTuple(docID, tuple);
+
+ connection.addToProcessor(op);
eventCounter.scope("Indexed").incrBy(1);
perSecMetrics.scope("Indexed").update(1);
-
- waitAck.addTuple(docID, tuple);
- } catch (IOException e) {
+ } catch (Exception e) {
LOG.error("Error building document for OpenSearch", e);
// do not send to status stream so that it gets replayed
_collector.fail(tuple);
+
waitAck.invalidate(docID);
}
}
@@ -230,7 +248,7 @@ public class IndexerBolt extends AbstractIndexerBolt implements BulkProcessor.Li
@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
eventCounter.scope("bulks_received").incrBy(1);
- eventCounter.scope("bulk_msec").incrBy(response.getTook().getMillis());
+ eventCounter.scope("bulk_msec").incrBy(response.took());
waitAck.processBulkResponse(
response,
@@ -240,7 +258,7 @@ public class IndexerBolt extends AbstractIndexerBolt implements BulkProcessor.Li
String url = (String) t.getValueByField("url");
Metadata metadata = (Metadata) t.getValueByField("metadata");
- if (!selected.failed) {
+ if (!selected.failed()) {
_collector.emit(
StatusStreamName, t, new Values(url, metadata, Status.FETCHED));
_collector.ack(t);
@@ -249,14 +267,16 @@ public class IndexerBolt extends AbstractIndexerBolt implements BulkProcessor.Li
LOG.error("update ID {}, URL {}, failure: {}", id, url, failure);
// there is something wrong with the content we should
// treat it as an ERROR
- if (failure.getStatus().equals(RestStatus.BAD_REQUEST)) {
+ if (selected.getStatus() == 400) {
metadata.setValue(Constants.STATUS_ERROR_SOURCE, "OpenSearch indexing");
metadata.setValue(Constants.STATUS_ERROR_MESSAGE, "invalid content");
_collector.emit(
StatusStreamName, t, new Values(url, metadata, Status.ERROR));
_collector.ack(t);
+ LOG.debug("Acked {} with ID {}", url, id);
} else {
_collector.fail(t);
+ LOG.debug("Failed {} with ID {}", url, id);
}
}
});
@@ -265,7 +285,6 @@ public class IndexerBolt extends AbstractIndexerBolt implements BulkProcessor.Li
@Override
public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
eventCounter.scope("bulks_received").incrBy(1);
-
waitAck.processFailedBulk(
request,
executionId,
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
index 345025fb..49ee7f0c 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java
@@ -63,10 +63,7 @@ public class JSONURLFilterWrapper extends URLFilter {
public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode filterParams) {
refresher =
new DelegateRefresher<>(
- URLFilter.class,
- stormConf,
- filterParams,
- (delegate, conf, params) -> delegate.configure(conf, params));
+ URLFilter.class, stormConf, filterParams, URLFilter::configure);
}
@Override
@@ -79,6 +76,8 @@ public class JSONURLFilterWrapper extends URLFilter {
@Override
public void cleanup() {
- refresher.cleanup();
+ if (refresher != null) {
+ refresher.cleanup();
+ }
}
}
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
index 6b9ccf4c..c46a2b73 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java
@@ -17,12 +17,11 @@
package org.apache.stormcrawler.opensearch.metrics;
-import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
-
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.Date;
+import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
@@ -32,8 +31,7 @@ import org.apache.storm.task.TopologyContext;
import org.apache.stormcrawler.opensearch.IndexCreation;
import org.apache.stormcrawler.opensearch.OpenSearchConnection;
import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.action.index.IndexRequest;
-import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.client.opensearch.core.bulk.BulkOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -144,19 +142,20 @@ public class MetricsConsumer implements IMetricsConsumer {
private void indexDataPoint(TaskInfo taskInfo, Date timestamp, String name, double value) {
try {
- XContentBuilder builder = jsonBuilder().startObject();
- builder.field("stormId", stormID);
- builder.field("srcComponentId", taskInfo.srcComponentId);
- builder.field("srcTaskId", taskInfo.srcTaskId);
- builder.field("srcWorkerHost", taskInfo.srcWorkerHost);
- builder.field("srcWorkerPort", taskInfo.srcWorkerPort);
- builder.field("name", name);
- builder.field("value", value);
- builder.field("timestamp", timestamp);
- builder.endObject();
-
- IndexRequest indexRequest = new IndexRequest(getIndexName(timestamp)).source(builder);
- connection.addToProcessor(indexRequest);
+ Map<String, Object> doc = new HashMap<>();
+ doc.put("stormId", stormID);
+ doc.put("srcComponentId", taskInfo.srcComponentId);
+ doc.put("srcTaskId", taskInfo.srcTaskId);
+ doc.put("srcWorkerHost", taskInfo.srcWorkerHost);
+ doc.put("srcWorkerPort", taskInfo.srcWorkerPort);
+ doc.put("name", name);
+ doc.put("value", value);
+ doc.put("timestamp", timestamp.toInstant().toString());
+
+ final String targetIndex = getIndexName(timestamp);
+ BulkOperation op =
+ BulkOperation.of(b -> b.index(idx -> idx.index(targetIndex).document(doc)));
+ connection.addToProcessor(op);
} catch (Exception e) {
LOG.error("problem when building request for OpenSearch", e);
}
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java
index 3957ba6a..d3b654a0 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java
@@ -17,8 +17,6 @@
package org.apache.stormcrawler.opensearch.metrics;
-import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
-
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
@@ -30,6 +28,7 @@ import com.codahale.metrics.Timer;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.SortedMap;
@@ -38,8 +37,7 @@ import org.apache.storm.metrics2.reporters.ScheduledStormReporter;
import org.apache.stormcrawler.opensearch.IndexCreation;
import org.apache.stormcrawler.opensearch.OpenSearchConnection;
import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.action.index.IndexRequest;
-import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.client.opensearch.core.bulk.BulkOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -68,6 +66,8 @@ public class MetricsReporter extends ScheduledStormReporter {
private static final String DATE_FORMAT_KEY = "opensearch.metrics.date.format";
+ private ScheduledReporter reporter;
+
@Override
public void prepare(
MetricRegistry metricsRegistry,
@@ -107,8 +107,6 @@ public class MetricsReporter extends ScheduledStormReporter {
reporter.start(reportPeriod, reportPeriodUnit);
}
- private ScheduledReporter reporter;
-
@Override
public void start() {
// already started in prepare()
@@ -123,7 +121,7 @@ public class MetricsReporter extends ScheduledStormReporter {
/**
* Inner ScheduledReporter that writes Codahale metrics to OpenSearch in the same format as the
- * V1 MetricsConsumer.
+ * V1 {@link MetricsConsumer}.
*/
private static class OpenSearchScheduledReporter extends ScheduledReporter {
@@ -203,16 +201,16 @@ public class MetricsReporter extends ScheduledStormReporter {
private void indexDataPoint(Date timestamp, String name, double value) {
try {
- XContentBuilder builder = jsonBuilder().startObject();
- builder.field("stormId", stormId);
- builder.field("name", name);
- builder.field("value", value);
- builder.field("timestamp", timestamp);
- builder.endObject();
-
- IndexRequest indexRequest =
- new IndexRequest(getIndexName(timestamp)).source(builder);
- connection.addToProcessor(indexRequest);
+ Map<String, Object> doc = new HashMap<>();
+ doc.put("stormId", stormId);
+ doc.put("name", name);
+ doc.put("value", value);
+ doc.put("timestamp", timestamp.toInstant().toString());
+
+ final String targetIndex = getIndexName(timestamp);
+ BulkOperation op =
+ BulkOperation.of(b -> b.index(idx -> idx.index(targetIndex).document(doc)));
+ connection.addToProcessor(op);
} catch (Exception e) {
LOG.error("Problem when building request for OpenSearch", e);
}
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
index 0b71e101..c74184c2 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java
@@ -17,8 +17,11 @@
package org.apache.stormcrawler.opensearch.metrics;
-import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.storm.Config;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
@@ -30,12 +33,8 @@ import org.apache.stormcrawler.metrics.CrawlerMetrics;
import org.apache.stormcrawler.opensearch.Constants;
import org.apache.stormcrawler.opensearch.OpenSearchConnection;
import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.RequestOptions;
-import org.opensearch.client.core.CountRequest;
-import org.opensearch.client.core.CountResponse;
-import org.opensearch.core.action.ActionListener;
-import org.opensearch.index.query.QueryBuilders;
-import org.opensearch.search.builder.SearchSourceBuilder;
+import org.opensearch.client.opensearch.OpenSearchClient;
+import org.opensearch.client.opensearch._types.FieldValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,45 +52,22 @@ public class StatusMetricsBolt extends BaseRichBolt {
private String indexName;
- private OpenSearchConnection connection;
+ private OpenSearchClient client;
- private Map<String, Long> latestStatusCounts = new HashMap<>(6);
+ private Map<String, Long> latestStatusCounts = new ConcurrentHashMap<>(6);
private int freqStats = 60;
private OutputCollector _collector;
- private transient StatusActionListener[] listeners;
+ private transient StatusCounter[] counters;
- private class StatusActionListener implements ActionListener<CountResponse> {
+ private static final class StatusCounter {
+ final String name;
+ final AtomicBoolean ready = new AtomicBoolean(true);
- private final String name;
-
- private boolean ready = true;
-
- public boolean isReady() {
- return ready;
- }
-
- public void busy() {
- this.ready = false;
- }
-
- StatusActionListener(String statusName) {
- name = statusName;
- }
-
- @Override
- public void onResponse(CountResponse response) {
- ready = true;
- LOG.debug("Got {} counts for status:{}", response.getCount(), name);
- latestStatusCounts.put(name, response.getCount());
- }
-
- @Override
- public void onFailure(Exception e) {
- ready = true;
- LOG.error("Failure when getting counts for status:{}", name, e);
+ StatusCounter(String name) {
+ this.name = name;
}
}
@@ -101,7 +77,7 @@ public class StatusMetricsBolt extends BaseRichBolt {
_collector = collector;
indexName = ConfUtils.getString(stormConf, OSStatusIndexNameParamName, "status");
try {
- connection = OpenSearchConnection.getConnection(stormConf, OSBoltType);
+ client = OpenSearchConnection.getClient(stormConf, OSBoltType);
} catch (Exception e1) {
LOG.error("Can't connect to OpenSearch", e1);
throw new RuntimeException(e1);
@@ -110,14 +86,14 @@ public class StatusMetricsBolt extends BaseRichBolt {
CrawlerMetrics.registerGauge(
context, stormConf, "status.count", () -> latestStatusCounts, freqStats);
- listeners = new StatusActionListener[6];
+ counters = new StatusCounter[6];
- listeners[0] = new StatusActionListener("DISCOVERED");
- listeners[1] = new StatusActionListener("FETCHED");
- listeners[2] = new StatusActionListener("FETCH_ERROR");
- listeners[3] = new StatusActionListener("REDIRECTION");
- listeners[4] = new StatusActionListener("ERROR");
- listeners[5] = new StatusActionListener("TOTAL");
+ counters[0] = new StatusCounter("DISCOVERED");
+ counters[1] = new StatusCounter("FETCHED");
+ counters[2] = new StatusCounter("FETCH_ERROR");
+ counters[3] = new StatusCounter("REDIRECTION");
+ counters[4] = new StatusCounter("ERROR");
+ counters[5] = new StatusCounter("TOTAL");
}
@Override
@@ -137,26 +113,69 @@ public class StatusMetricsBolt extends BaseRichBolt {
return;
}
- for (StatusActionListener listener : listeners) {
+ for (StatusCounter counter : counters) {
// still waiting for results from previous request
- if (!listener.isReady()) {
- LOG.debug("Not ready to get counts for status {}", listener.name);
+ if (!counter.ready.compareAndSet(true, false)) {
+ LOG.debug("Not ready to get counts for status {}", counter.name);
continue;
}
- CountRequest request = new CountRequest(indexName);
- if (!listener.name.equalsIgnoreCase("TOTAL")) {
- SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
- sourceBuilder.query(QueryBuilders.termQuery("status", listener.name));
- request.source(sourceBuilder);
- }
- listener.busy();
- connection.getClient().countAsync(request, RequestOptions.DEFAULT, listener);
+ final String statusName = counter.name;
+ CompletableFuture.supplyAsync(
+ () -> {
+ try {
+ if (statusName.equalsIgnoreCase("TOTAL")) {
+ return client.count(c -> c.index(indexName));
+ } else {
+ return client.count(
+ c ->
+ c.index(indexName)
+ .query(
+ q ->
+ q.term(
+ t ->
+ t.field(
+ "status")
+ .value(
+ FieldValue
+ .of(
+ statusName)))));
+ }
+ } catch (Exception e) {
+ throw new CompletionException(e);
+ }
+ })
+ .thenAccept(
+ response -> {
+ counter.ready.set(true);
+ LOG.debug(
+ "Got {} counts for status:{}",
+ response.count(),
+ statusName);
+ latestStatusCounts.put(statusName, response.count());
+ })
+ .exceptionally(
+ e -> {
+ counter.ready.set(true);
+ Throwable cause =
+ e instanceof CompletionException ? e.getCause() : e;
+ LOG.error(
+ "Failure when getting counts for status:{}",
+ statusName,
+ cause);
+ return null;
+ });
}
}
@Override
public void cleanup() {
- connection.close();
+ if (client != null) {
+ try {
+ client._transport().close();
+ } catch (Exception e) {
+ LOG.error("Exception closing client transport", e);
+ }
+ }
}
@Override
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
index d8995ff8..6f6d7be6 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java
@@ -62,10 +62,7 @@ public class JSONResourceWrapper extends ParseFilter {
public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode filterParams) {
refresher =
new DelegateRefresher<>(
- ParseFilter.class,
- stormConf,
- filterParams,
- (delegate, conf, params) -> delegate.configure(conf, params));
+ ParseFilter.class, stormConf, filterParams, ParseFilter::configure);
}
@Override
@@ -75,6 +72,8 @@ public class JSONResourceWrapper extends ParseFilter {
@Override
public void cleanup() {
- refresher.cleanup();
+ if (refresher != null) {
+ refresher.cleanup();
+ }
}
}
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
index b29ebd16..0afe2b19 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java
@@ -19,10 +19,15 @@ package org.apache.stormcrawler.opensearch.persistence;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Date;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.stormcrawler.Metadata;
@@ -31,8 +36,8 @@ import org.apache.stormcrawler.opensearch.IndexCreation;
import org.apache.stormcrawler.opensearch.OpenSearchConnection;
import org.apache.stormcrawler.persistence.AbstractQueryingSpout;
import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.client.RestHighLevelClient;
-import org.opensearch.search.SearchHit;
+import org.opensearch.client.json.JsonData;
+import org.opensearch.client.opensearch.OpenSearchClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -72,7 +77,7 @@ public abstract class AbstractSpout extends AbstractQueryingSpout {
protected String indexName;
- protected static RestHighLevelClient client;
+ protected static OpenSearchClient client;
/**
* when using multiple instances - each one is in charge of a specific shard useful when
@@ -98,6 +103,15 @@ public abstract class AbstractSpout extends AbstractQueryingSpout {
protected int queryTimeout = -1;
+ /**
+ * Single-threaded executor dedicated to running the asynchronous OpenSearch queries triggered
+ * from this spout. Using a dedicated, named, daemon executor avoids polluting the JVM-wide
+ * {@link java.util.concurrent.ForkJoinPool#commonPool() common pool} (which is shared with all
+ * other Storm tasks running in the same worker) and allows clean shutdown when the topology is
+ * stopped.
+ */
+ protected ExecutorService queryExecutor;
+
@Override
public void open(
Map<String, Object> stormConf,
@@ -106,6 +120,19 @@ public abstract class AbstractSpout extends AbstractQueryingSpout {
super.open(stormConf, context, collector);
+ final String executorName =
+ "OpenSearchSpout-"
+ + context.getThisComponentId()
+ + "-"
+ + context.getThisTaskIndex();
+ queryExecutor =
+ Executors.newSingleThreadExecutor(
+ r -> {
+ Thread t = new Thread(r, executorName);
+ t.setDaemon(true);
+ return t;
+ });
+
indexName = ConfUtils.getString(stormConf, OSStatusIndexNameParamName, "status");
// one OS client per JVM
@@ -180,14 +207,59 @@ public abstract class AbstractSpout extends AbstractQueryingSpout {
/** Builds a query and use it retrieve the results from OS * */
protected abstract void populateBuffer();
- protected final boolean addHitToBuffer(SearchHit hit) {
- Map<String, Object> keyValues = hit.getSourceAsMap();
- String url = (String) keyValues.get("url");
+ /**
+ * Converts the {@link JsonData} carried by a search hit into a plain {@code Map<String,
+ * Object>} populated with native Java types (strings, numbers, lists, nested maps).
+ *
+ * <p>This helper exists to work around a subtle short-circuit inside {@code JsonDataImpl#to}:
+ *
+ * <pre>{@code
+ * if (clazz.isAssignableFrom(value.getClass())) {
+ * return (T) value;
+ * }
+ * }</pre>
+ *
+ * <p>When {@link JsonData} is produced via {@code JsonData._DESERIALIZER} (as opposed to a raw
+ * Jackson {@code TreeNode}), its internal {@code value} is a Jakarta JSON-P {@code JsonObject}
+ * — typically a Parsson {@code JsonObjectImpl}. Crucially, {@code jakarta.json.JsonObject
+ * extends Map<String, JsonValue>}, so calling {@code source.to(Map.class)} or {@code
+ * source.to(Object.class)} matches the {@code isAssignableFrom} guard and returns the raw
+ * {@code JsonObject} <em>without ever consulting the configured {@link
+ * org.opensearch.client.json.jackson.JacksonJsonpMapper}</em>. The resulting map values are
+ * still {@code JsonString}/{@code JsonStringImpl} instances and any downstream {@code (String)
+ * value} cast blows up with a {@link ClassCastException}.
+ *
+ * <p>Passing a concrete subclass that {@code JsonObject} does <em>not</em> implement (here
+ * {@link LinkedHashMap}) bypasses the short-circuit and forces the round-trip through {@code
+ * JacksonJsonpGenerator -> JacksonJsonpParser -> Jackson ObjectMapper.readValue}, which yields
+ * native Java values (String, Long, Double, ArrayList, LinkedHashMap…).
+ *
+ * @param source the document source carried by an OpenSearch hit; may be {@code null}
+ * @return a (possibly empty) map of native Java values, never {@code null}
+ */
+ static Map<String, Object> sourceAsMap(JsonData source) {
+ if (source == null) {
+ return Collections.emptyMap();
+ }
+ @SuppressWarnings("unchecked")
+ Map<String, Object> map = source.to(LinkedHashMap.class);
+ return map == null ? Collections.emptyMap() : map;
+ }
+
+ /**
+ * Adds a document source to the URL buffer unless it is already being processed.
+ *
+ * @param source the document source as a key-value map (must contain a "url" entry)
+ * @return {@code true} if the URL was added to the buffer, {@code false} if it was already
+ * being processed or already present
+ */
+ protected final boolean addHitToBuffer(Map<String, Object> source) {
+ String url = (String) source.get("url");
// is already being processed - skip it!
if (beingProcessed.containsKey(url)) {
return false;
}
- return buffer.add(url, fromKeyValues(keyValues));
+ return buffer.add(url, fromKeyValues(source));
}
protected final Metadata fromKeyValues(Map<String, Object> keyValues) {
@@ -225,10 +297,29 @@ public abstract class AbstractSpout extends AbstractQueryingSpout {
@Override
public void close() {
+ // Shut the dedicated query executor down first so no more requests can be enqueued
+ // against the client while we are tearing it down. We give in-flight queries a short
+ // grace period to drain to keep shutdown reasonably graceful.
+ if (queryExecutor != null) {
+ queryExecutor.shutdown();
+ try {
+ if (!queryExecutor.awaitTermination(30, TimeUnit.SECONDS)) {
+ LOG.warn(
+ "{} Query executor did not terminate within 30s, forcing shutdown",
+ logIdprefix);
+ queryExecutor.shutdownNow();
+ }
+ } catch (InterruptedException e) {
+ queryExecutor.shutdownNow();
+ Thread.currentThread().interrupt();
+ }
+ queryExecutor = null;
+ }
+
synchronized (AbstractSpout.class) {
if (client != null) {
try {
- client.close();
+ client._transport().close();
} catch (IOException e) {
LOG.error("Exception caught when closing client", e);
}
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java
index 79936b36..a6272f36 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java
@@ -17,45 +17,38 @@
package org.apache.stormcrawler.opensearch.persistence;
-import static org.opensearch.index.query.QueryBuilders.boolQuery;
-
+import java.io.IOException;
import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
import java.util.Calendar;
+import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
import org.apache.commons.lang3.StringUtils;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.opensearch.Constants;
import org.apache.stormcrawler.util.ConfUtils;
-import org.joda.time.format.ISODateTimeFormat;
-import org.opensearch.action.search.SearchRequest;
-import org.opensearch.action.search.SearchResponse;
-import org.opensearch.client.RequestOptions;
-import org.opensearch.core.action.ActionListener;
-import org.opensearch.index.query.BoolQueryBuilder;
-import org.opensearch.index.query.QueryBuilders;
-import org.opensearch.search.SearchHit;
-import org.opensearch.search.aggregations.AggregationBuilders;
-import org.opensearch.search.aggregations.Aggregations;
-import org.opensearch.search.aggregations.BucketOrder;
-import org.opensearch.search.aggregations.bucket.SingleBucketAggregation;
-import org.opensearch.search.aggregations.bucket.sampler.DiversifiedAggregationBuilder;
-import org.opensearch.search.aggregations.bucket.terms.Terms;
-import org.opensearch.search.aggregations.bucket.terms.Terms.Bucket;
-import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
-import org.opensearch.search.aggregations.metrics.TopHits;
-import org.opensearch.search.builder.SearchSourceBuilder;
-import org.opensearch.search.sort.FieldSortBuilder;
-import org.opensearch.search.sort.SortBuilders;
-import org.opensearch.search.sort.SortOrder;
+import org.opensearch.client.json.JsonData;
+import org.opensearch.client.opensearch._types.FieldValue;
+import org.opensearch.client.opensearch._types.SortOrder;
+import org.opensearch.client.opensearch._types.aggregations.Aggregate;
+import org.opensearch.client.opensearch._types.aggregations.Aggregation;
+import org.opensearch.client.opensearch._types.aggregations.StringTermsBucket;
+import org.opensearch.client.opensearch._types.aggregations.TopHitsAggregate;
+import org.opensearch.client.opensearch.core.SearchRequest;
+import org.opensearch.client.opensearch.core.SearchResponse;
+import org.opensearch.client.opensearch.core.search.Hit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,7 +58,7 @@ import org.slf4j.LoggerFactory;
* the same number of spout instances as OpenSearch shards. Guarantees a good mix of URLs by
* aggregating them by an arbitrary field e.g. key.
*/
-public class AggregationSpout extends AbstractSpout implements ActionListener<SearchResponse> {
+public class AggregationSpout extends AbstractSpout {
private static final Logger LOG = LoggerFactory.getLogger(AggregationSpout.class);
@@ -104,106 +97,171 @@ public class AggregationSpout extends AbstractSpout implements ActionListener<Se
lastTimeResetToNow = Instant.now();
}
- String formattedQueryDate = ISODateTimeFormat.dateTimeNoMillis().print(queryDate.getTime());
+ String formattedQueryDate =
+ Instant.ofEpochMilli(queryDate.getTime())
+ .atOffset(ZoneOffset.UTC)
+ .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
LOG.info("{} Populating buffer with nextFetchDate <= {}", logIdprefix, formattedQueryDate);
- BoolQueryBuilder queryBuilder =
- boolQuery()
- .filter(QueryBuilders.rangeQuery("nextFetchDate").lte(formattedQueryDate));
-
- if (filterQueries != null) {
- for (String filterQuery : filterQueries) {
- queryBuilder.filter(QueryBuilders.queryStringQuery(filterQuery));
- }
+ // Build the top_hits sub-aggregation
+ Aggregation topHitsAgg =
+ Aggregation.of(
+ a ->
+ a.topHits(
+ th -> {
+ th.size(maxURLsPerBucket).explain(false);
+ for (String bsf : bucketSortField) {
+ th.sort(
+ s ->
+ s.field(
+ fs ->
+ fs.field(bsf)
+ .order(
+ SortOrder
+ .Asc)));
+ }
+ return th;
+ }));
+
+ // Build the terms (partition) aggregation with top_hits sub-agg
+ Aggregation.Builder.ContainerBuilder partitionAggBuilder =
+ new Aggregation.Builder()
+ .terms(
+ t -> {
+ t.field(partitionField).size(maxBucketNum);
+ // sort between buckets by the min sub-aggregation
+ if (StringUtils.isNotBlank(totalSortField)) {
+ t.order(
+ Collections.singletonList(
+ Collections.singletonMap(
+ "top_hit", SortOrder.Asc)));
+ }
+ return t;
+ })
+ .aggregations("docs", topHitsAgg);
+
+ // add the min sub-aggregation used for sorting between buckets
+ if (StringUtils.isNotBlank(totalSortField)) {
+ partitionAggBuilder.aggregations(
+ "top_hit", Aggregation.of(minAgg -> minAgg.min(m -> m.field(totalSortField))));
}
- SearchRequest request = new SearchRequest(indexName);
-
- SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
- sourceBuilder.query(queryBuilder);
- sourceBuilder.from(0);
- sourceBuilder.size(0);
- sourceBuilder.explain(false);
- sourceBuilder.trackTotalHits(false);
+ Aggregation partitionAgg = partitionAggBuilder.build();
+
+ // Build the search request
+ SearchRequest.Builder requestBuilder =
+ new SearchRequest.Builder()
+ .index(indexName)
+ .size(0)
+ .trackTotalHits(t -> t.enabled(false))
+ .query(
+ q ->
+ q.bool(
+ b -> {
+ b.filter(
+ f ->
+ f.range(
+ r ->
+ r.field(
+ "nextFetchDate")
+ .lte(
+ JsonData
+ .of(
+ formattedQueryDate))));
+ if (filterQueries != null) {
+ for (String fq : filterQueries) {
+ b.filter(
+ f ->
+ f.queryString(
+ qs ->
+ qs
+ .query(
+ fq)));
+ }
+ }
+ return b;
+ }));
if (queryTimeout != -1) {
- sourceBuilder.timeout(
- new org.opensearch.common.unit.TimeValue(queryTimeout, TimeUnit.SECONDS));
- }
-
- TermsAggregationBuilder aggregations =
- AggregationBuilders.terms("partition").field(partitionField).size(maxBucketNum);
-
- org.opensearch.search.aggregations.metrics.TopHitsAggregationBuilder tophits =
- AggregationBuilders.topHits("docs").size(maxURLsPerBucket).explain(false);
-
- // sort within a bucket
- for (String bsf : bucketSortField) {
- FieldSortBuilder sorter = SortBuilders.fieldSort(bsf).order(SortOrder.ASC);
- tophits.sort(sorter);
- }
-
- aggregations.subAggregation(tophits);
-
- // sort between buckets
- if (StringUtils.isNotBlank(totalSortField)) {
- org.opensearch.search.aggregations.metrics.MinAggregationBuilder minBuilder =
- AggregationBuilders.min("top_hit").field(totalSortField);
- aggregations.subAggregation(minBuilder);
- aggregations.order(BucketOrder.aggregation("top_hit", true));
+ requestBuilder.timeout(queryTimeout + "s");
}
if (sample) {
- DiversifiedAggregationBuilder sab = new DiversifiedAggregationBuilder("sample");
- sab.field(partitionField).maxDocsPerValue(maxURLsPerBucket);
- sab.shardSize(maxURLsPerBucket * maxBucketNum);
- sab.subAggregation(aggregations);
- sourceBuilder.aggregation(sab);
+ // Wrap in a diversified sampler aggregation
+ requestBuilder.aggregations(
+ "sample",
+ Aggregation.of(
+ a ->
+ a.diversifiedSampler(
+ ds ->
+ ds.field(partitionField)
+ .maxDocsPerValue(
+ maxURLsPerBucket)
+ .shardSize(
+ maxURLsPerBucket
+ * maxBucketNum))
+ .aggregations("partition", partitionAgg)));
} else {
- sourceBuilder.aggregation(aggregations);
+ requestBuilder.aggregations("partition", partitionAgg);
}
- request.source(sourceBuilder);
-
- // https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-preference.html
- // _shards:2,3
- // specific shard but ideally a local copy of it
+ // shard preference for routing
if (shardID != -1) {
- request.preference("_shards:" + shardID + "|_local");
+ requestBuilder.preference("_shards:" + shardID + "|_local");
}
+ SearchRequest request = requestBuilder.build();
+
// dump query to log
LOG.debug("{} OpenSearch query {}", logIdprefix, request);
LOG.trace("{} isInQuery set to true", logIdprefix);
isInQuery.set(true);
- client.searchAsync(request, RequestOptions.DEFAULT, this);
- }
- @Override
- public void onFailure(Exception arg0) {
- LOG.error("{} Exception with OpenSearch query", logIdprefix, arg0);
- markQueryReceivedNow();
+ CompletableFuture.supplyAsync(
+ () -> {
+ try {
+ return client.search(request, JsonData.class);
+ } catch (IOException e) {
+ throw new CompletionException(e);
+ }
+ },
+ queryExecutor)
+ .thenAccept(this::handleResponse)
+ .exceptionally(
+ e -> {
+ Throwable cause = e instanceof CompletionException ? e.getCause() : e;
+ LOG.error("{} Exception with OpenSearch query", logIdprefix, cause);
+ markQueryReceivedNow();
+ return null;
+ });
}
- @Override
- public void onResponse(SearchResponse response) {
+ /**
+ * Handles the search response from an asynchronous aggregation query, extracting URLs from term
+ * buckets and adding them to the buffer.
+ *
+ * @param response the search response containing aggregation results
+ */
+ protected void handleResponse(SearchResponse<JsonData> response) {
long timeTaken = System.currentTimeMillis() - getTimeLastQuerySent();
- Aggregations aggregs = response.getAggregations();
+ Map<String, Aggregate> aggregs = response.aggregations();
- if (aggregs == null) {
+ if (aggregs == null || aggregs.isEmpty()) {
markQueryReceivedNow();
return;
}
- SingleBucketAggregation sample = aggregs.get("sample");
- if (sample != null) {
- aggregs = sample.getAggregations();
+ // Unwrap the sample aggregation if present
+ Aggregate sampleAgg = aggregs.get("sample");
+ if (sampleAgg != null) {
+ aggregs = sampleAgg.sampler().aggregations();
}
- Terms agg = aggregs.get("partition");
+ Aggregate partitionAgg = aggregs.get("partition");
+ List<StringTermsBucket> buckets = partitionAgg.sterms().buckets().array();
int numhits = 0;
int numBuckets = 0;
@@ -214,35 +272,31 @@ public class AggregationSpout extends AbstractSpout implements ActionListener<Se
currentBuckets.clear();
// For each entry
- Iterator<Terms.Bucket> iterator = (Iterator<Bucket>) agg.getBuckets().iterator();
+ Iterator<StringTermsBucket> iterator = buckets.iterator();
while (iterator.hasNext()) {
- Terms.Bucket entry = iterator.next();
- String key = (String) entry.getKey(); // bucket key
+ StringTermsBucket entry = iterator.next();
+ String key = entry.key(); // bucket key
currentBuckets.add(key);
- long docCount = entry.getDocCount(); // Doc count
+ long docCount = entry.docCount(); // Doc count
int hitsForThisBucket = 0;
- SearchHit lastHit = null;
-
+ List<FieldValue> lastSortValues = null;
// filter results so that we don't include URLs we are already
// being processed
- TopHits topHits = entry.getAggregations().get("docs");
- for (SearchHit hit : topHits.getHits().getHits()) {
+ TopHitsAggregate topHits = entry.aggregations().get("docs").topHits();
+ for (Hit<JsonData> hit : topHits.hits().hits()) {
- LOG.debug(
- "{} -> id [{}], _source [{}]",
- logIdprefix,
- hit.getId(),
- hit.getSourceAsString());
+ Map<String, Object> keyValues = sourceAsMap(hit.source());
+
+ LOG.debug("{} -> id [{}], _source [{}]", logIdprefix, hit.id(), keyValues);
hitsForThisBucket++;
- lastHit = hit;
+ lastSortValues = hit.sort();
- Map<String, Object> keyValues = hit.getSourceAsMap();
String url = (String) keyValues.get("url");
// consider only the first document of the last bucket
@@ -273,8 +327,8 @@ public class AggregationSpout extends AbstractSpout implements ActionListener<Se
LOG.debug("{} -> added to buffer : {}", logIdprefix, url);
}
- if (lastHit != null) {
- sortValuesForKey(key, lastHit.getSortValues());
+ if (lastSortValues != null && !lastSortValues.isEmpty()) {
+ sortValuesForKey(key, lastSortValues.toArray());
}
if (hitsForThisBucket > 0) {
@@ -369,5 +423,10 @@ public class AggregationSpout extends AbstractSpout implements ActionListener<Se
markQueryReceivedNow();
}
+ /**
+ * Receives the sort values of the last hit returned for a bucket, as the {@link FieldValue}
+ * instances of the client. Subclasses can pass them to {@code search_after} to read the bucket
+ * from that position.
+ */
protected void sortValuesForKey(String key, Object[] sortValues) {}
}
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
index 551153f5..b3301e5c 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java
@@ -17,31 +17,28 @@
package org.apache.stormcrawler.opensearch.persistence;
-import static org.opensearch.index.query.QueryBuilders.boolQuery;
-
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
+import java.io.IOException;
import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
import java.util.Date;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.stormcrawler.opensearch.Constants;
import org.apache.stormcrawler.persistence.EmptyQueueListener;
import org.apache.stormcrawler.util.ConfUtils;
-import org.joda.time.format.ISODateTimeFormat;
-import org.opensearch.action.search.SearchRequest;
-import org.opensearch.action.search.SearchResponse;
-import org.opensearch.client.RequestOptions;
-import org.opensearch.core.action.ActionListener;
-import org.opensearch.index.query.BoolQueryBuilder;
-import org.opensearch.index.query.QueryBuilders;
-import org.opensearch.search.SearchHit;
-import org.opensearch.search.builder.SearchSourceBuilder;
-import org.opensearch.search.sort.FieldSortBuilder;
-import org.opensearch.search.sort.SortBuilders;
-import org.opensearch.search.sort.SortOrder;
+import org.opensearch.client.json.JsonData;
+import org.opensearch.client.opensearch._types.FieldValue;
+import org.opensearch.client.opensearch._types.SortOrder;
+import org.opensearch.client.opensearch.core.SearchRequest;
+import org.opensearch.client.opensearch.core.SearchResponse;
+import org.opensearch.client.opensearch.core.search.Hit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -90,7 +87,7 @@ public class HybridSpout extends AggregationSpout implements EmptyQueueListener
// would just overload OpenSearch and yield
// mainly duplicates
if (isInQuery.get()) {
- LOG.trace("{} isInquery true", logIdprefix, queueName);
+ LOG.trace("{} isInquery true for {}", logIdprefix, queueName);
return;
}
@@ -101,57 +98,89 @@ public class HybridSpout extends AggregationSpout implements EmptyQueueListener
lastTimeResetToNow = Instant.now();
}
- String formattedQueryDate = ISODateTimeFormat.dateTimeNoMillis().print(queryDate.getTime());
-
- BoolQueryBuilder queryBuilder =
- boolQuery()
- .filter(QueryBuilders.rangeQuery("nextFetchDate").lte(formattedQueryDate));
-
- queryBuilder.filter(QueryBuilders.termQuery(partitionField, queueName));
-
- SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
- sourceBuilder.query(queryBuilder);
- sourceBuilder.from(0);
- sourceBuilder.size(bufferReloadSize);
- sourceBuilder.explain(false);
- sourceBuilder.trackTotalHits(false);
+ String formattedQueryDate =
+ Instant.ofEpochMilli(queryDate.getTime())
+ .atOffset(ZoneOffset.UTC)
+ .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
+
+ SearchRequest.Builder requestBuilder =
+ new SearchRequest.Builder()
+ .index(indexName)
+ .size(bufferReloadSize)
+ .trackTotalHits(t -> t.enabled(false))
+ .query(
+ q ->
+ q.bool(
+ b ->
+ b.filter(
+ f ->
+ f.range(
+ r ->
+ r.field(
+ "nextFetchDate")
+ .lte(
+ JsonData
+ .of(
+ formattedQueryDate))))
+ .filter(
+ f ->
+ f.term(
+ t ->
+ t.field(
+ partitionField)
+ .value(
+ FieldValue
+ .of(
+ queueName))))));
// sort within a bucket
for (String bsf : bucketSortField) {
- FieldSortBuilder sorter = SortBuilders.fieldSort(bsf).order(SortOrder.ASC);
- sourceBuilder.sort(sorter);
+ requestBuilder.sort(s -> s.field(fs -> fs.field(bsf).order(SortOrder.Asc)));
}
// do we have a search after for this one?
Object[] searchAfterValues = searchAfterCache.getIfPresent(queueName);
if (searchAfterValues != null) {
- sourceBuilder.searchAfter(searchAfterValues);
+ for (Object sav : searchAfterValues) {
+ requestBuilder.searchAfter((FieldValue) sav);
+ }
}
- SearchRequest request = new SearchRequest(indexName);
-
- request.source(sourceBuilder);
-
- // https://www.elastic.co/guide/en/opensearch/reference/current/search-request-preference.html
- // _shards:2,3
- // specific shard but ideally a local copy of it
+ // shard preference for routing
if (shardID != -1) {
- request.preference("_shards:" + shardID + "|_local");
+ requestBuilder.preference("_shards:" + shardID + "|_local");
}
- // dump query to log
- LOG.debug("{} OpenSearch query {} - {}", logIdprefix, queueName, request.toString());
+ SearchRequest request = requestBuilder.build();
- client.searchAsync(request, RequestOptions.DEFAULT, hrl);
+ // dump query to log
+ LOG.debug("{} OpenSearch query {} - {}", logIdprefix, queueName, request);
+
+ CompletableFuture.supplyAsync(
+ () -> {
+ try {
+ return client.search(request, JsonData.class);
+ } catch (IOException e) {
+ throw new CompletionException(e);
+ }
+ },
+ queryExecutor)
+ .thenAccept(hrl::handleResponse)
+ .exceptionally(
+ e -> {
+ Throwable cause = e instanceof CompletionException ? e.getCause() : e;
+ LOG.error("Exception with OpenSearch query", cause);
+ return null;
+ });
}
/** Overrides the handling of responses for aggregations. */
@Override
- public void onResponse(SearchResponse response) {
+ protected void handleResponse(SearchResponse<JsonData> response) {
// delete all entries from the searchAfterCache when
// we get the results from the aggregation spouts
searchAfterCache.invalidateAll();
- super.onResponse(response);
+ super.handleResponse(response);
}
/** The aggregation kindly told us where to start from. */
@@ -163,40 +192,52 @@ public class HybridSpout extends AggregationSpout implements EmptyQueueListener
}
/** Handling of results for a specific queue. */
- class HostResultListener implements ActionListener<SearchResponse> {
+ class HostResultListener {
- @Override
- public void onResponse(SearchResponse response) {
+ /**
+ * Handles the search response for a host-specific query, extracting hits and adding them to
+ * the buffer.
+ *
+ * @param response the search response containing document hits
+ */
+ void handleResponse(SearchResponse<JsonData> response) {
int alreadyprocessed = 0;
int numDocs = 0;
- SearchHit[] hits = response.getHits().getHits();
+ List<Hit<JsonData>> hits = response.hits().hits();
Object[] sortValues = null;
// retrieve the key for these results
String key = null;
- for (SearchHit hit : hits) {
+ for (Hit<JsonData> hit : hits) {
numDocs++;
+
+ Map<String, Object> source = sourceAsMap(hit.source());
+
String pfield = partitionField;
- Map<String, Object> sourceAsMap = hit.getSourceAsMap();
+ Map<String, Object> fieldSource = source;
if (pfield.startsWith("metadata.")) {
- sourceAsMap = (Map<String, Object>) sourceAsMap.get("metadata");
+ @SuppressWarnings("unchecked")
+ Map<String, Object> metadataMap = (Map<String, Object>) source.get("metadata");
+ fieldSource = metadataMap;
pfield = pfield.substring(9);
}
- Object key_as_object = sourceAsMap.get(pfield);
+ Object key_as_object = fieldSource.get(pfield);
if (key_as_object instanceof List) {
- if (((List<String>) (key_as_object)).size() == 1) {
- key = ((List<String>) key_as_object).get(0);
+ @SuppressWarnings("unchecked")
+ List<String> keyList = (List<String>) key_as_object;
+ if (keyList.size() == 1) {
+ key = keyList.get(0);
}
} else {
key = key_as_object.toString();
}
- sortValues = hit.getSortValues();
- if (!addHitToBuffer(hit)) {
+ sortValues = hit.sort().toArray();
+ if (!addHitToBuffer(source)) {
alreadyprocessed++;
}
}
@@ -214,14 +255,9 @@ public class HybridSpout extends AggregationSpout implements EmptyQueueListener
"{} OpenSearch term query returned {} hits in {} msec with {} already being processed for {}",
logIdprefix,
numDocs,
- response.getTook().getMillis(),
+ response.took(),
alreadyprocessed,
key);
}
-
- @Override
- public void onFailure(Exception e) {
- LOG.error("Exception with OpenSearch query", e);
- }
}
}
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java
index 3e144872..6af7d909 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java
@@ -29,17 +29,15 @@ import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.opensearch.AsyncBulkProcessor;
import org.apache.stormcrawler.opensearch.Constants;
import org.apache.stormcrawler.opensearch.IndexCreation;
import org.apache.stormcrawler.opensearch.OpenSearchConnection;
import org.apache.stormcrawler.util.ConfUtils;
-import org.opensearch.action.DocWriteRequest;
-import org.opensearch.action.bulk.BulkItemResponse;
-import org.opensearch.action.bulk.BulkProcessor;
-import org.opensearch.action.bulk.BulkRequest;
-import org.opensearch.action.bulk.BulkResponse;
-import org.opensearch.action.index.IndexRequest;
-import org.opensearch.core.rest.RestStatus;
+import org.opensearch.client.opensearch.core.BulkRequest;
+import org.opensearch.client.opensearch.core.BulkResponse;
+import org.opensearch.client.opensearch.core.bulk.BulkOperation;
+import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,7 +45,7 @@ import org.slf4j.LoggerFactory;
* Sends information about the queues into an OpenSearch index. This has to be connected to a status
* updater bolt.
*/
-public class QueueBolt extends BaseRichBolt implements BulkProcessor.Listener {
+public class QueueBolt extends BaseRichBolt implements AsyncBulkProcessor.Listener {
private static final Logger LOG = LoggerFactory.getLogger(QueueBolt.class);
@@ -140,36 +138,39 @@ public class QueueBolt extends BaseRichBolt implements BulkProcessor.Listener {
fields.put("key", key);
fields.put("lastUpdated", Instant.now().toString());
- final IndexRequest indexRequest =
- new IndexRequest(indexName).source(fields).id(docID).create(true);
+ final BulkOperation op =
+ BulkOperation.of(b -> b.create(c -> c.index(indexName).id(docID).document(fields)));
- connection.addToProcessor(indexRequest);
+ connection.addToProcessor(op);
}
@Override
public void beforeBulk(long executionId, BulkRequest request) {
- LOG.debug("beforeBulk {} with {} actions", executionId, request.numberOfActions());
+ LOG.debug("beforeBulk {} with {} actions", executionId, request.operations().size());
}
@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
- LOG.debug("afterBulk [{}] with {} responses", executionId, request.numberOfActions());
- for (BulkItemResponse item : response.getItems()) {
- final String key = inFlight.remove(item.getId());
+ LOG.debug("afterBulk [{}] with {} responses", executionId, response.items().size());
+ for (BulkResponseItem item : response.items()) {
+ final String key = inFlight.remove(item.id());
if (key == null) {
continue;
}
- if (!item.isFailed()) {
+ if (item.error() == null) {
knownQueue.put(key, Boolean.TRUE);
- } else if (RestStatus.CONFLICT.equals(item.getFailure().getStatus())) {
+ } else if (item.status() == 409) {
// entry already exists in the index, e.g. written before a worker
// restart or evicted from the cache - not an error
knownQueue.put(key, Boolean.TRUE);
} else {
// do not cache the key so that the write is retried
// when the key is next seen
+ final var error = item.error();
LOG.error(
- "Failed to index queue entry {}: {}", key, item.getFailure().getMessage());
+ "Failed to index queue entry {}: {}",
+ key,
+ error.reason() != null ? error.reason() : error.type());
}
}
}
@@ -177,9 +178,9 @@ public class QueueBolt extends BaseRichBolt implements BulkProcessor.Listener {
@Override
public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
LOG.error("Exception with bulk {} - failing the whole lot ", executionId, failure);
- for (DocWriteRequest<?> docWriteRequest : request.requests()) {
+ for (BulkOperation op : request.operations()) {
// not cached - the writes are retried when the keys are next seen
- inFlight.remove(docWriteRequest.id());
+ inFlight.remove(OpenSearchConnection.getBulkOperationId(op));
}
}
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
index f025f582..2dcd76fe 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java
@@ -18,7 +18,9 @@
package org.apache.stormcrawler.opensearch.persistence;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Date;
+import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
@@ -31,6 +33,7 @@ import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.metrics.CrawlerMetrics;
import org.apache.stormcrawler.metrics.ScopedCounter;
import org.apache.stormcrawler.metrics.ScopedReducedMetric;
+import org.apache.stormcrawler.opensearch.AsyncBulkProcessor;
import org.apache.stormcrawler.opensearch.Constants;
import org.apache.stormcrawler.opensearch.IndexCreation;
import org.apache.stormcrawler.opensearch.OpenSearchConnection;
@@ -39,12 +42,9 @@ import org.apache.stormcrawler.persistence.AbstractStatusUpdaterBolt;
import org.apache.stormcrawler.persistence.Status;
import org.apache.stormcrawler.util.ConfUtils;
import org.apache.stormcrawler.util.URLPartitioner;
-import org.opensearch.action.bulk.BulkProcessor;
-import org.opensearch.action.bulk.BulkRequest;
-import org.opensearch.action.bulk.BulkResponse;
-import org.opensearch.action.index.IndexRequest;
-import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.client.opensearch.core.BulkRequest;
+import org.opensearch.client.opensearch.core.BulkResponse;
+import org.opensearch.client.opensearch.core.bulk.BulkOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,7 +52,8 @@ import org.slf4j.LoggerFactory;
* Simple bolt which stores the status of URLs into OpenSearch. Takes the tuples coming from the
* 'status' stream. To be used in combination with a Spout to read from the index.
*/
-public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt implements BulkProcessor.Listener {
+public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt
+ implements AsyncBulkProcessor.Listener {
private static final Logger LOG = LoggerFactory.getLogger(StatusUpdaterBolt.class);
@@ -142,6 +143,16 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt implements Bulk
int metrics_time_bucket_secs = 30;
+ // benchmarking - average number of items received back from OpenSearch per second
+ this.receivedPerSecMetrics =
+ CrawlerMetrics.registerPerSecMetric(
+ context, stormConf, "average_persec", metrics_time_bucket_secs);
+
+ // eventCounter MUST be registered before WaitAckCache — the eviction lambda captures it
+ this.eventCounter =
+ CrawlerMetrics.registerCounter(
+ context, stormConf, "counters", metrics_time_bucket_secs);
+
String defaultSpec =
String.format(
Locale.ROOT,
@@ -159,24 +170,13 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt implements Bulk
eventCounter.scope("purged").incrBy(1);
collector.fail(t);
});
-
- // create gauge for waitAck
CrawlerMetrics.registerGauge(
context, stormConf, "waitAck", waitAck::estimatedSize, metrics_time_bucket_secs);
- // benchmarking - average number of items received back by Elastic per second
- this.receivedPerSecMetrics =
- CrawlerMetrics.registerPerSecMetric(
- context, stormConf, "average_persec", metrics_time_bucket_secs);
-
- this.eventCounter =
- CrawlerMetrics.registerCounter(
- context, stormConf, "counters", metrics_time_bucket_secs);
-
try {
connection = OpenSearchConnection.getConnection(stormConf, OSBoltType, this);
} catch (Exception e1) {
- LOG.error("Can't connect to ElasticSearch", e1);
+ LOG.error("Can't connect to OpenSearch", e1);
throw new RuntimeException(e1);
}
@@ -205,8 +205,10 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt implements Bulk
String documentID = getDocumentID(metadata, url);
- // check that the same URL is not being sent to OpenSearch
- if (status.equals(Status.DISCOVERED) && waitAck.contains(documentID)) {
+ boolean isAlreadySentAndDiscovered =
+ status.equals(Status.DISCOVERED) && waitAck.contains(documentID);
+
+ if (isAlreadySentAndDiscovered) {
// if this object is discovered - adding another version of it
// won't make any difference
LOG.debug(
@@ -220,16 +222,16 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt implements Bulk
return;
}
- XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
- builder.field("url", url);
- builder.field("status", status);
+ Map<String, Object> doc = new HashMap<>();
+ doc.put("url", url);
+ doc.put("status", status.name());
- builder.startObject("metadata");
+ Map<String, Object> metadataMap = new HashMap<>();
for (String mdKey : metadata.keySet()) {
String[] values = metadata.getValues(mdKey);
// periods are not allowed - replace with %2E
mdKey = mdKey.replaceAll("\\.", "%2E");
- builder.array(mdKey, values);
+ metadataMap.put(mdKey, Arrays.asList(values));
}
String partitionKey = partitioner.getPartition(url, metadata);
@@ -245,62 +247,81 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt implements Bulk
// store routing key in metadata?
if (StringUtils.isNotBlank(fieldNameForRoutingKey) && routingFieldNameInMetadata) {
- builder.field(fieldNameForRoutingKey, partitionKey);
+ metadataMap.put(fieldNameForRoutingKey, partitionKey);
}
- builder.endObject();
+ doc.put("metadata", metadataMap);
// store routing key outside metadata?
if (StringUtils.isNotBlank(fieldNameForRoutingKey) && !routingFieldNameInMetadata) {
- builder.field(fieldNameForRoutingKey, partitionKey);
+ doc.put(fieldNameForRoutingKey, partitionKey);
}
if (nextFetch.isPresent()) {
- builder.timeField("nextFetchDate", nextFetch.get());
+ doc.put("nextFetchDate", nextFetch.get().toInstant().toString());
}
-
- builder.endObject();
-
- IndexRequest request = new IndexRequest(getIndexName(metadata));
-
// check that we don't overwrite an existing entry
// When create is used, the index operation will fail if a document
// by that id already exists in the index.
final boolean create = status.equals(Status.DISCOVERED);
- request.source(builder).id(documentID).create(create);
-
- if (doRouting) {
- request.routing(partitionKey);
+ final String targetIndex = getIndexName(metadata);
+ final String routing = doRouting ? partitionKey : null;
+
+ BulkOperation op;
+ if (create) {
+ op =
+ BulkOperation.of(
+ b ->
+ b.create(
+ c -> {
+ c.index(targetIndex).id(documentID).document(doc);
+ if (routing != null) {
+ c.routing(routing);
+ }
+ return c;
+ }));
+ } else {
+ op =
+ BulkOperation.of(
+ b ->
+ b.index(
+ idx -> {
+ idx.index(targetIndex).id(documentID).document(doc);
+ if (routing != null) {
+ idx.routing(routing);
+ }
+ return idx;
+ }));
}
waitAck.addTuple(documentID, tuple);
LOG.debug("Sending to OpenSearch buffer {} with ID {}", url, documentID);
- connection.addToProcessor(request);
+ connection.addToProcessor(op);
}
@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
- LOG.debug("afterBulk [{}] with {} responses", executionId, request.numberOfActions());
+ LOG.debug("afterBulk [{}] with {} responses", executionId, request.operations().size());
eventCounter.scope("bulks_received").incrBy(1);
- eventCounter.scope("bulk_msec").incrBy(response.getTook().getMillis());
- eventCounter.scope("received").incrBy(request.numberOfActions());
- receivedPerSecMetrics.scope("received").update(request.numberOfActions());
+ eventCounter.scope("bulk_msec").incrBy(response.took());
+ eventCounter.scope("received").incrBy(request.operations().size());
+ receivedPerSecMetrics.scope("received").update(request.operations().size());
waitAck.processBulkResponse(
response,
executionId,
eventCounter,
- (id, t, selected) -> {
- if (!selected.failed) {
- String url = t.getStringByField("url");
+ (id, tuple, selected) -> {
+ if (!selected.failed()) {
+ String url = tuple.getStringByField("url");
LOG.debug("Acked {} with ID {}", url, id);
eventCounter.scope("acked").incrBy(1);
- ack(t, url);
+ super.ack(tuple, url);
} else {
eventCounter.scope("failed").incrBy(1);
- collector.fail(t);
+ collector.fail(tuple);
}
});
}
@@ -308,8 +329,8 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt implements Bulk
@Override
public void afterBulk(long executionId, BulkRequest request, Throwable throwable) {
eventCounter.scope("bulks_received").incrBy(1);
- eventCounter.scope("received").incrBy(request.numberOfActions());
- receivedPerSecMetrics.scope("received").update(request.numberOfActions());
+ eventCounter.scope("received").incrBy(request.operations().size());
+ receivedPerSecMetrics.scope("received").update(request.operations().size());
waitAck.processFailedBulk(
request,
@@ -323,7 +344,7 @@ public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt implements Bulk
@Override
public void beforeBulk(long executionId, BulkRequest request) {
- LOG.debug("beforeBulk {} with {} actions", executionId, request.numberOfActions());
+ LOG.debug("beforeBulk {} with {} actions", executionId, request.operations().size());
eventCounter.scope("bulks_sent").incrBy(1);
}
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorHttpTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorHttpTest.java
similarity index 100%
rename from external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorHttpTest.java
rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorHttpTest.java
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorRetryTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorRetryTest.java
similarity index 100%
rename from external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorRetryTest.java
rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorRetryTest.java
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorTest.java
similarity index 100%
rename from external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorTest.java
rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorTest.java
diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java
index ae90bebd..bb484e84 100644
--- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java
@@ -23,186 +23,95 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.UncheckedIOException;
-import java.net.InetAddress;
-import java.net.ServerSocket;
-import java.net.Socket;
-import java.nio.charset.StandardCharsets;
import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
-import org.apache.http.HttpHost;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.Credentials;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
+import org.apache.hc.client5.http.auth.AuthScope;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
+import org.apache.hc.core5.http.HttpHost;
import org.junit.jupiter.api.Test;
-import org.opensearch.client.Request;
-import org.opensearch.client.RestClient;
-import org.opensearch.client.sniff.OpenSearchNodesSniffer;
class OpenSearchConnectionCredentialsTest {
- private static final Credentials CREDENTIALS =
- new UsernamePasswordCredentials("crawler", "s3cret");
-
- private static CredentialsProvider providerFor(HttpHost... hosts) {
- return new OpenSearchConnection.OriginCredentialsProvider(List.of(hosts), CREDENTIALS);
- }
-
- /** The scope HttpClient looks the credentials up with for a request to the given node. */
- private static AuthScope request(String scheme, String host, int port) {
- return new AuthScope(new HttpHost(host, port, scheme), AuthScope.ANY_REALM, "Basic");
+ private static BasicCredentialsProvider providerFor(List<HttpHost> hosts) {
+ final BasicCredentialsProvider provider = new BasicCredentialsProvider();
+ final UsernamePasswordCredentials credentials =
+ new UsernamePasswordCredentials("crawler", "s3cret".toCharArray());
+ for (AuthScope scope : OpenSearchConnection.credentialScopes(hosts)) {
+ provider.setCredentials(scope, credentials);
+ }
+ return provider;
}
@Test
- void credentialsAreGivenToTheConfiguredAddresses() {
- final CredentialsProvider provider =
- providerFor(
- new HttpHost("opensearch1.example.org", 9200, "https"),
- new HttpHost("OpenSearch2.example.org", 9201, "https"));
- assertEquals(
- CREDENTIALS,
- provider.getCredentials(request("https", "opensearch1.example.org", 9200)));
+ void scopesCoverTheConfiguredHostsAndPorts() {
+ final List<AuthScope> scopes =
+ OpenSearchConnection.credentialScopes(
+ List.of(
+ new HttpHost("https", "opensearch1.example.org", 9200),
+ new HttpHost("https", "opensearch2.example.org", 9201)));
assertEquals(
- CREDENTIALS,
- provider.getCredentials(request("HTTPS", "opensearch2.example.org", 9201)));
+ List.of(
+ new AuthScope("https", "opensearch1.example.org", 9200, null, null),
+ new AuthScope("https", "opensearch2.example.org", 9201, null, null)),
+ scopes);
}
@Test
- void credentialsAreOnlyGivenToTheConfiguredHosts() {
- final CredentialsProvider provider =
- providerFor(new HttpHost("opensearch1.example.org", 9200, "https"));
-
- // a node found by the sniffer under the address it publishes
- assertNull(provider.getCredentials(request("https", "10.0.0.12", 9200)));
- // same host, another port
- assertNull(provider.getCredentials(request("https", "opensearch1.example.org", 9300)));
- assertNull(provider.getCredentials(request("https", "other.example.org", 9200)));
- }
-
- @Test
- void credentialsAreMatchedOnTheScheme() {
- final CredentialsProvider provider =
- providerFor(new HttpHost("opensearch1.example.org", 9200, "https"));
- assertNull(provider.getCredentials(request("http", "opensearch1.example.org", 9200)));
+ void duplicateAddressesGiveOneScope() {
+ final List<AuthScope> scopes =
+ OpenSearchConnection.credentialScopes(
+ List.of(
+ new HttpHost("https", "opensearch1.example.org", 9200),
+ new HttpHost("https", "OpenSearch1.example.org", 9200)));
+ assertEquals(1, scopes.size());
}
@Test
- void credentialsNeedAnOrigin() {
- final CredentialsProvider provider =
- providerFor(new HttpHost("opensearch1.example.org", 9200, "https"));
- assertNull(provider.getCredentials(new AuthScope("opensearch1.example.org", 9200)));
- assertNull(provider.getCredentials(AuthScope.ANY));
+ void noAddressGivesNoScope() {
+ assertTrue(OpenSearchConnection.credentialScopes(List.of()).isEmpty());
}
@Test
- void noAddressGivesTheCredentialsToNoOne() {
- assertNull(providerFor().getCredentials(request("https", "opensearch1.example.org", 9200)));
- }
+ void credentialsAreOnlyGivenToTheConfiguredHosts() {
+ final BasicCredentialsProvider provider =
+ providerFor(List.of(new HttpHost("https", "opensearch1.example.org", 9200)));
- /**
- * Sends a request with the client to a local server and returns the Authorization header it
- * received, if any.
- */
- private static String authorizationSentTo(HttpHost configured) throws Exception {
- try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getLoopbackAddress())) {
- final int port = server.getLocalPort();
- final CompletableFuture<String> authorization =
- CompletableFuture.supplyAsync(
- () -> {
- try (Socket socket = server.accept()) {
- final BufferedReader in =
- new BufferedReader(
- new InputStreamReader(
- socket.getInputStream(),
- StandardCharsets.US_ASCII));
- String header = null;
- for (String line = in.readLine();
- line != null && !line.isEmpty();
- line = in.readLine()) {
- if (line.regionMatches(true, 0, "Authorization:", 0, 14)) {
- header = line.substring(14).trim();
- }
- }
- socket.getOutputStream()
- .write(
- "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
- .getBytes(StandardCharsets.US_ASCII));
- return header;
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- });
- final HttpHost configuredHere =
- new HttpHost(configured.getHostName(), port, configured.getSchemeName());
- try (RestClient client =
- RestClient.builder(new HttpHost("127.0.0.1", port, "http"))
- .setHttpClientConfigCallback(
- b ->
- b.setDefaultCredentialsProvider(
- providerFor(configuredHere)))
- .build()) {
- client.performRequest(new Request("GET", "/"));
- }
- return authorization.get(10, TimeUnit.SECONDS);
- }
+ assertNotNull(
+ provider.getCredentials(request("https", "opensearch1.example.org", 9200), null));
+ // another node of the cluster which is not listed in the addresses
+ assertNull(provider.getCredentials(request("https", "10.0.0.12", 9200), null));
+ // same host, another port
+ assertNull(
+ provider.getCredentials(request("https", "opensearch1.example.org", 9300), null));
+ assertNull(provider.getCredentials(request("https", "other.example.org", 9200), null));
}
@Test
- void clientSendsTheCredentialsToAConfiguredAddress() throws Exception {
- assertNotNull(authorizationSentTo(new HttpHost("127.0.0.1", 0, "http")));
+ void credentialsAreMatchedOnTheScheme() {
+ final BasicCredentialsProvider provider =
+ providerFor(List.of(new HttpHost("https", "opensearch1.example.org", 9200)));
+ assertNull(provider.getCredentials(request("http", "opensearch1.example.org", 9200), null));
}
- @Test
- void clientSendsNoCredentialsToAnotherAddress() throws Exception {
- assertNull(authorizationSentTo(new HttpHost("127.0.0.1", 0, "https")));
+ /** The scope HttpClient looks the credentials up with for a request to the given node. */
+ private static AuthScope request(String scheme, String host, int port) {
+ return new AuthScope(new HttpHost(scheme, host, port), null, "Basic");
}
@Test
void plainHttpToARemoteHostIsReported() {
- final HttpHost remote = new HttpHost("opensearch1.example.org", 9200, "http");
+ final HttpHost remote = new HttpHost("http", "opensearch1.example.org", 9200);
final List<HttpHost> plain =
OpenSearchConnection.plainHttpHosts(
List.of(
remote,
- new HttpHost("opensearch2.example.org", 9200, "https"),
- new HttpHost("localhost", 9200, "http"),
- new HttpHost("127.0.0.1", 9200, "http"),
- new HttpHost("[::1]", 9200, "http")));
+ new HttpHost("https", "opensearch2.example.org", 9200),
+ new HttpHost("http", "localhost", 9200),
+ new HttpHost("http", "127.0.0.1", 9200),
+ new HttpHost("http", "[::1]", 9200)));
assertEquals(List.of(remote), plain);
}
- @Test
- void sniffedNodesKeepTheSchemeOfTheAddresses() {
- assertEquals(
- OpenSearchNodesSniffer.Scheme.HTTPS,
- OpenSearchConnection.sniffScheme(
- List.of(new HttpHost("opensearch1.example.org", 9200, "https"))));
- assertEquals(
- OpenSearchNodesSniffer.Scheme.HTTPS,
- OpenSearchConnection.sniffScheme(
- List.of(
- new HttpHost("opensearch1.example.org", 9200, "http"),
- new HttpHost("opensearch2.example.org", 9200, "https"))));
- assertEquals(
- OpenSearchNodesSniffer.Scheme.HTTP,
- OpenSearchConnection.sniffScheme(
- List.of(new HttpHost("opensearch1.example.org", 9200, "http"))));
- }
-
- @Test
- void credentialsNeedUserAndPassword() {
- assertTrue(OpenSearchConnection.hasCredentials("crawler", "s3cret"));
- assertFalse(OpenSearchConnection.hasCredentials("crawler", null));
- assertFalse(OpenSearchConnection.hasCredentials("crawler", " "));
- assertFalse(OpenSearchConnection.hasCredentials(null, "s3cret"));
- assertFalse(OpenSearchConnection.hasCredentials("", ""));
- }
-
@Test
void loopbackAddresses() {
assertTrue(OpenSearchConnection.isLoopback("localhost"));
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java
similarity index 100%
rename from external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java
rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java
diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java
index 0fd22d18..3bd2dc6d 100644
--- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java
@@ -17,31 +17,29 @@
package org.apache.stormcrawler.opensearch;
+import static org.awaitility.Awaitility.await;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import com.github.benmanes.caffeine.cache.Ticker;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
import org.apache.storm.tuple.Tuple;
import org.apache.stormcrawler.metrics.ScopedCounter;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import org.opensearch.action.DocWriteRequest;
-import org.opensearch.action.bulk.BulkItemResponse;
-import org.opensearch.action.bulk.BulkRequest;
-import org.opensearch.action.bulk.BulkResponse;
-import org.opensearch.action.delete.DeleteRequest;
-import org.opensearch.action.index.IndexResponse;
-import org.opensearch.core.index.shard.ShardId;
-import org.opensearch.core.rest.RestStatus;
+import org.opensearch.client.opensearch._types.ErrorCause;
+import org.opensearch.client.opensearch.core.BulkRequest;
+import org.opensearch.client.opensearch.core.BulkResponse;
+import org.opensearch.client.opensearch.core.bulk.BulkOperation;
+import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
+import org.opensearch.client.opensearch.core.bulk.OperationType;
import org.slf4j.LoggerFactory;
class WaitAckCacheTest {
@@ -66,23 +64,33 @@ class WaitAckCacheTest {
return t;
}
- private static ShardId shardId() {
- return new ShardId("index", "_na_", 0);
+ private static BulkResponseItem successItem(String docId) {
+ return BulkResponseItem.of(
+ b -> b.id(docId).index("index").status(200).operationType(OperationType.Index));
}
- private static BulkItemResponse successItem(int itemId, String docId) {
- IndexResponse indexResponse = new IndexResponse(shardId(), docId, 1, 1, 1, true);
- return new BulkItemResponse(itemId, DocWriteRequest.OpType.INDEX, indexResponse);
+ private static BulkResponseItem failedItem(String docId, int status) {
+ return BulkResponseItem.of(
+ b ->
+ b.id(docId)
+ .index("index")
+ .status(status)
+ .operationType(OperationType.Index)
+ .error(
+ ErrorCause.of(
+ e -> e.type("test_error").reason("test failure"))));
}
- private static BulkItemResponse failedItem(int itemId, String docId, RestStatus status) {
- BulkItemResponse.Failure failure =
- new BulkItemResponse.Failure("index", docId, new Exception("test failure"), status);
- return new BulkItemResponse(itemId, DocWriteRequest.OpType.INDEX, failure);
- }
-
- private static BulkResponse bulkResponse(BulkItemResponse... items) {
- return new BulkResponse(items, 10L);
+ private static BulkResponse bulkResponse(BulkResponseItem... items) {
+ boolean hasErrors = false;
+ for (BulkResponseItem item : items) {
+ if (item.error() != null) {
+ hasErrors = true;
+ break;
+ }
+ }
+ final boolean errors = hasErrors;
+ return BulkResponse.of(b -> b.took(10).errors(errors).items(List.of(items)));
}
@Test
@@ -110,14 +118,14 @@ class WaitAckCacheTest {
Tuple t = mockTuple("http://example.com");
cache.addTuple("doc1", t);
- BulkResponse response = bulkResponse(successItem(0, "doc1"));
+ BulkResponse response = bulkResponse(successItem("doc1"));
cache.processBulkResponse(
response,
1L,
null,
(id, tuple, selected) -> {
- if (!selected.failed) {
+ if (!selected.failed()) {
acked.add(tuple);
} else {
failed.add(tuple);
@@ -135,15 +143,14 @@ class WaitAckCacheTest {
Tuple t = mockTuple("http://example.com");
cache.addTuple("doc1", t);
- BulkResponse response =
- bulkResponse(failedItem(0, "doc1", RestStatus.INTERNAL_SERVER_ERROR));
+ BulkResponse response = bulkResponse(failedItem("doc1", 500));
cache.processBulkResponse(
response,
1L,
null,
(id, tuple, selected) -> {
- if (!selected.failed) {
+ if (!selected.failed()) {
acked.add(tuple);
} else {
failed.add(tuple);
@@ -160,15 +167,18 @@ class WaitAckCacheTest {
Tuple t = mockTuple("http://example.com");
cache.addTuple("doc1", t);
- ScopedCounter counter = scopeName -> incrementBy -> {};
- BulkResponse response = bulkResponse(failedItem(0, "doc1", RestStatus.CONFLICT));
+ ScopedCounter counter = mock(ScopedCounter.class);
+ ScopedCounter.CountHandle handle = mock(ScopedCounter.CountHandle.class);
+ when(counter.scope("doc_conflicts")).thenReturn(handle);
+
+ BulkResponse response = bulkResponse(failedItem("doc1", 409));
cache.processBulkResponse(
response,
1L,
counter,
(id, tuple, selected) -> {
- if (!selected.failed) {
+ if (!selected.failed()) {
acked.add(tuple);
} else {
failed.add(tuple);
@@ -177,6 +187,7 @@ class WaitAckCacheTest {
assertEquals(1, acked.size());
assertEquals(0, failed.size());
+ verify(handle).incrBy(1);
}
@Test
@@ -186,7 +197,7 @@ class WaitAckCacheTest {
cache.addTuple("doc1", t1);
cache.addTuple("doc1", t2);
- BulkResponse response = bulkResponse(successItem(0, "doc1"));
+ BulkResponse response = bulkResponse(successItem("doc1"));
cache.processBulkResponse(response, 1L, null, (id, tuple, selected) -> acked.add(tuple));
@@ -201,17 +212,14 @@ class WaitAckCacheTest {
Tuple t = mockTuple("http://example.com");
cache.addTuple("doc1", t);
- BulkResponse response =
- bulkResponse(
- failedItem(0, "doc1", RestStatus.INTERNAL_SERVER_ERROR),
- successItem(1, "doc1"));
+ BulkResponse response = bulkResponse(failedItem("doc1", 500), successItem("doc1"));
cache.processBulkResponse(
response,
1L,
null,
(id, tuple, selected) -> {
- if (!selected.failed) {
+ if (!selected.failed()) {
acked.add(tuple);
} else {
failed.add(tuple);
@@ -229,9 +237,14 @@ class WaitAckCacheTest {
cache.addTuple("doc1", t1);
cache.addTuple("doc2", t2);
- BulkRequest request = new BulkRequest();
- request.add(new DeleteRequest("index", "doc1"));
- request.add(new DeleteRequest("index", "doc2"));
+ BulkRequest request =
+ BulkRequest.of(
+ b ->
+ b.operations(
+ BulkOperation.of(
+ o -> o.delete(d -> d.index("index").id("doc1"))),
+ BulkOperation.of(
+ o -> o.delete(d -> d.index("index").id("doc2")))));
cache.processFailedBulk(request, 1L, new Exception("connection lost"), failed::add);
@@ -247,8 +260,17 @@ class WaitAckCacheTest {
Tuple t = mockTuple("http://example.com");
cache.addTuple("doc1", t);
- BulkRequest request = new BulkRequest();
- request.add(new DeleteRequest("index", "doc_unknown"));
+ BulkRequest request =
+ BulkRequest.of(
+ b ->
+ b.operations(
+ BulkOperation.of(
+ o ->
+ o.delete(
+ d ->
+ d.index("index")
+ .id(
+ "doc_unknown")))));
cache.processFailedBulk(request, 1L, new Exception("test"), failed::add);
@@ -259,25 +281,29 @@ class WaitAckCacheTest {
@Test
void eviction_failsTuplesOnExpiry() {
- AtomicLong fakeTime = new AtomicLong(0);
- Ticker fakeTicker = fakeTime::get;
cache =
new WaitAckCache(
"expireAfterWrite=1s",
LoggerFactory.getLogger(WaitAckCacheTest.class),
- evicted::add,
- fakeTicker);
+ evicted::add);
Tuple t = mockTuple("http://example.com");
cache.addTuple("doc1", t);
- // Advance past the 1s expiry
- fakeTime.set(TimeUnit.SECONDS.toNanos(2));
-
- // Access triggers expiration; cleanUp forces listener execution
- cache.contains("doc1");
- cache.cleanUp();
+ // Force cache maintenance after expiry by doing a contains() check
+ // which accesses the cache and triggers Caffeine's cleanup
+ await().atMost(5, TimeUnit.SECONDS)
+ .pollInterval(200, TimeUnit.MILLISECONDS)
+ .untilAsserted(
+ () -> {
+ // contains() accesses the cache which triggers cleanup
+ cache.contains("doc1");
+ // also try adding and invalidating a dummy entry to force maintenance
+ Tuple dummy = mockTuple("http://dummy");
+ cache.addTuple("_probe_", dummy);
+ cache.invalidate("_probe_");
+ assertFalse(evicted.isEmpty(), "Eviction callback should have fired");
+ });
- assertFalse(evicted.isEmpty(), "Eviction callback should have fired");
assertTrue(evicted.contains(t));
}
@@ -288,17 +314,14 @@ class WaitAckCacheTest {
cache.addTuple("doc1", t1);
cache.addTuple("doc2", t2);
- BulkResponse response =
- bulkResponse(
- successItem(0, "doc1"),
- failedItem(1, "doc2", RestStatus.INTERNAL_SERVER_ERROR));
+ BulkResponse response = bulkResponse(successItem("doc1"), failedItem("doc2", 500));
cache.processBulkResponse(
response,
1L,
null,
(id, tuple, selected) -> {
- if (!selected.failed) {
+ if (!selected.failed()) {
acked.add(tuple);
} else {
failed.add(tuple);
@@ -310,20 +333,4 @@ class WaitAckCacheTest {
assertEquals(1, failed.size());
assertSame(t2, failed.get(0));
}
-
- @Test
- void shutdown_failsAllRemainingTuples() {
- Tuple t1 = mockTuple("http://example.com/1");
- Tuple t2 = mockTuple("http://example.com/2");
- cache.addTuple("doc1", t1);
- cache.addTuple("doc2", t2);
-
- cache.shutdown();
-
- assertEquals(2, evicted.size());
- assertTrue(evicted.contains(t1));
- assertTrue(evicted.contains(t2));
- assertFalse(cache.contains("doc1"));
- assertFalse(cache.contains("doc2"));
- }
}
diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
index e9c72b33..44150ea6 100644
--- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java
@@ -26,7 +26,8 @@ import org.testcontainers.utility.DockerImageName;
@Testcontainers(disabledWithoutDocker = true)
public abstract class AbstractOpenSearchTest {
- private static final String OPENSEARCH_VERSION = "2.19.4";
+ private static final String OPENSEARCH_VERSION =
+ System.getProperty("opensearch-version", "2.19.5");
public static final String PASSWORD = "This1sAPassw0rd";
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java
similarity index 100%
rename from external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java
rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java
diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java
index 92b665c5..296c58dd 100644
--- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java
@@ -31,7 +31,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import org.apache.http.HttpHost;
+import org.apache.hc.core5.http.HttpHost;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
import org.apache.stormcrawler.Metadata;
@@ -44,12 +44,11 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
-import org.opensearch.action.get.GetRequest;
-import org.opensearch.action.get.GetResponse;
-import org.opensearch.client.RequestOptions;
-import org.opensearch.client.RestClient;
-import org.opensearch.client.RestClientBuilder;
-import org.opensearch.client.RestHighLevelClient;
+import org.opensearch.client.json.jackson.JacksonJsonpMapper;
+import org.opensearch.client.opensearch.OpenSearchClient;
+import org.opensearch.client.opensearch.core.GetResponse;
+import org.opensearch.client.transport.OpenSearchTransport;
+import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -59,7 +58,9 @@ class QueueBoltTest extends AbstractOpenSearchTest {
protected TestOutputCollector output;
- protected org.opensearch.client.RestHighLevelClient client;
+ protected OpenSearchClient client;
+
+ private OpenSearchTransport transport;
private static final Logger LOG = LoggerFactory.getLogger(QueueBoltTest.class);
@@ -78,12 +79,15 @@ class QueueBoltTest extends AbstractOpenSearchTest {
@BeforeEach
void setupQueueBolt() throws IOException {
- RestClientBuilder builder =
- RestClient.builder(
- new HttpHost(
- opensearchContainer.getHost(),
- opensearchContainer.getMappedPort(9200)));
- client = new RestHighLevelClient(builder);
+ transport =
+ ApacheHttpClient5TransportBuilder.builder(
+ new HttpHost(
+ "http",
+ opensearchContainer.getHost(),
+ opensearchContainer.getMappedPort(9200)))
+ .setMapper(new JacksonJsonpMapper())
+ .build();
+ client = new OpenSearchClient(transport);
output = new TestOutputCollector();
bolt = prepareQueueBolt();
}
@@ -105,7 +109,7 @@ class QueueBoltTest extends AbstractOpenSearchTest {
bolt.cleanup();
output = null;
try {
- client.close();
+ transport.close();
} catch (IOException e) {
}
}
@@ -129,20 +133,19 @@ class QueueBoltTest extends AbstractOpenSearchTest {
.until(
() -> {
try {
- GetResponse result =
- client.get(
- new GetRequest("queues", id),
- RequestOptions.DEFAULT);
- return result.isExists();
+ GetResponse<Map> result =
+ client.get(g -> g.index("queues").id(id), Map.class);
+ return result.found();
} catch (IOException e) {
return false;
}
});
}
+ @SuppressWarnings("unchecked")
private Map<String, Object> getSource(String key) throws IOException {
String id = org.apache.commons.codec.digest.DigestUtils.sha256Hex(key);
- return client.get(new GetRequest("queues", id), RequestOptions.DEFAULT).getSourceAsMap();
+ return client.get(g -> g.index("queues").id(id), Map.class).source();
}
@Test
diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java
index 19df1453..71fe7795 100644
--- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java
+++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java
@@ -32,7 +32,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import org.apache.http.HttpHost;
+import org.apache.hc.core5.http.HttpHost;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
import org.apache.stormcrawler.Metadata;
@@ -46,12 +46,11 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
-import org.opensearch.action.get.GetRequest;
-import org.opensearch.action.get.GetResponse;
-import org.opensearch.client.RequestOptions;
-import org.opensearch.client.RestClient;
-import org.opensearch.client.RestClientBuilder;
-import org.opensearch.client.RestHighLevelClient;
+import org.opensearch.client.json.jackson.JacksonJsonpMapper;
+import org.opensearch.client.opensearch.OpenSearchClient;
+import org.opensearch.client.opensearch.core.GetResponse;
+import org.opensearch.client.transport.OpenSearchTransport;
+import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,7 +60,9 @@ class StatusBoltTest extends AbstractOpenSearchTest {
protected TestOutputCollector output;
- protected org.opensearch.client.RestHighLevelClient client;
+ protected OpenSearchClient client;
+
+ private OpenSearchTransport transport;
private static final Logger LOG = LoggerFactory.getLogger(StatusBoltTest.class);
@@ -81,12 +82,15 @@ class StatusBoltTest extends AbstractOpenSearchTest {
@BeforeEach
void setupStatusBolt() throws IOException {
bolt = new StatusUpdaterBolt();
- RestClientBuilder builder =
- RestClient.builder(
- new HttpHost(
- opensearchContainer.getHost(),
- opensearchContainer.getMappedPort(9200)));
- client = new RestHighLevelClient(builder);
+ transport =
+ ApacheHttpClient5TransportBuilder.builder(
+ new HttpHost(
+ "http",
+ opensearchContainer.getHost(),
+ opensearchContainer.getMappedPort(9200)))
+ .setMapper(new JacksonJsonpMapper())
+ .build();
+ client = new OpenSearchClient(transport);
// configure the status updater bolt
Map<String, Object> conf = new HashMap<>();
conf.put("opensearch.status.routing.fieldname", "metadata.key");
@@ -107,7 +111,7 @@ class StatusBoltTest extends AbstractOpenSearchTest {
bolt.cleanup();
output = null;
try {
- client.close();
+ transport.close();
} catch (IOException e) {
}
}
@@ -129,6 +133,7 @@ class StatusBoltTest extends AbstractOpenSearchTest {
@Test
@Timeout(value = 2, unit = TimeUnit.MINUTES)
// see https://github.com/apache/stormcrawler/issues/885
+ @SuppressWarnings("unchecked")
void checkListKeyFromOpensearch()
throws IOException, ExecutionException, InterruptedException, TimeoutException {
String url = "https://www.url.net/something";
@@ -136,10 +141,10 @@ class StatusBoltTest extends AbstractOpenSearchTest {
md.addValue("someKey", "someValue");
store(url, Status.DISCOVERED, md).get(10, TimeUnit.SECONDS);
assertEquals(1, output.getAckedTuples().size());
- // check output in Opensearch?
+ // check output in Opensearch
String id = org.apache.commons.codec.digest.DigestUtils.sha256Hex(url);
- GetResponse result = client.get(new GetRequest("status", id), RequestOptions.DEFAULT);
- Map<String, Object> sourceAsMap = result.getSourceAsMap();
+ GetResponse<Map> result = client.get(g -> g.index("status").id(id), Map.class);
+ Map<String, Object> sourceAsMap = result.source();
final String pfield = "metadata.somekey";
sourceAsMap = (Map<String, Object>) sourceAsMap.get("metadata");
final var pfieldNew = pfield.substring(9);
diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpoutTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpoutTest.java
similarity index 100%
rename from external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpoutTest.java
rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpoutTest.java