You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by jn...@apache.org on 2024/11/01 13:22:17 UTC
(incubator-stormcrawler) 01/01: Detect multi addresses separated by ; in a single String. Fixes #1382
This is an automated email from the ASF dual-hosted git repository.
jnioche pushed a commit to branch 1382
in repository https://gitbox.apache.org/repos/asf/incubator-stormcrawler.git
commit 376fbfaac05daf6c9ddc683fa48ad6289e72e700
Author: Julien Nioche <ju...@digitalpebble.com>
AuthorDate: Fri Nov 1 13:21:57 2024 +0000
Detect multi addresses separated by ; in a single String. Fixes #1382
Signed-off-by: Julien Nioche <ju...@digitalpebble.com>
---
.../src/main/resources/archetype-resources/opensearch-conf.yaml | 2 ++
external/opensearch/opensearch-conf.yaml | 2 ++
.../org/apache/stormcrawler/opensearch/OpenSearchConnection.java | 9 +++++++++
3 files changed, 13 insertions(+)
diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml b/external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml
index 4181f131..1b3342cf 100644
--- a/external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml
+++ b/external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml
@@ -19,6 +19,8 @@ config:
# address to use unless a more specific one has been
# defined for a component
+ # also accepts a list or multiple values in a single line
+ # separated by a semi-colon e.g. "opensearch1:9200; opensearch2:9200"
opensearch.addresses: "http://localhost:9200"
#opensearch.user: "USERNAME"
#opensearch.password: "PASSWORD"
diff --git a/external/opensearch/opensearch-conf.yaml b/external/opensearch/opensearch-conf.yaml
index 8ec0341c..d70c9d02 100644
--- a/external/opensearch/opensearch-conf.yaml
+++ b/external/opensearch/opensearch-conf.yaml
@@ -19,6 +19,8 @@ config:
# address to use unless a more specific one has been
# defined for a component
+ # also accepts a list or multiple values in a single line
+ # separated by a semi-colon e.g. "opensearch1:9200; opensearch2:9200"
opensearch.addresses: "http://localhost:9200"
#opensearch.user: "USERNAME"
#opensearch.password: "PASSWORD"
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
index a5c5da46..0db6cd7f 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java
@@ -22,6 +22,7 @@ import static org.opensearch.client.RestClientBuilder.DEFAULT_SOCKET_TIMEOUT_MIL
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@@ -89,6 +90,14 @@ public final class OpenSearchConnection {
ConfUtils.loadListFromConf(
Constants.PARAMPREFIX, dottedType, "addresses", stormConf);
+ // find ; separated values and tokenise as multiple addresses
+ // e.g. opensearch1:9200; opensearch2:9200
+ if (confighosts.size() == 1) {
+ String input = confighosts.get(0);
+ confighosts.clear();
+ confighosts.addAll(Arrays.asList(input.split(" *; *")));
+ }
+
for (String host : confighosts) {
// no port specified? use default one
int port = 9200;