You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by nf...@apache.org on 2023/06/01 09:36:24 UTC
[camel] branch main updated: CAMEL-19400: camel-elasticsearch - Use singleton service to speed up (#10252)
This is an automated email from the ASF dual-hosted git repository.
nfilotto pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new d7927e12d94 CAMEL-19400: camel-elasticsearch - Use singleton service to speed up (#10252)
d7927e12d94 is described below
commit d7927e12d94e28e07b88fe6436317d7f00941c40
Author: Nicolas Filotto <es...@users.noreply.github.com>
AuthorDate: Thu Jun 1 11:36:16 2023 +0200
CAMEL-19400: camel-elasticsearch - Use singleton service to speed up (#10252)
## Motivation
The build on Jenkins takes between 7h and 8h which is much too long especially when we know that the max duration allowed is about 9 h.
## Modifications:
* Adds a `SingletonService` to the `ElasticSearchServiceFactory`
* Adds the management of the authentication to the `ElasticSearchService`
* Adds the integration tests to use a singleton service to start only once an ES instance
---
.../integration/ElasticsearchScrollSearchIT.java | 2 +-
.../es/integration/ElasticsearchSizeLimitIT.java | 6 +-
.../es/integration/ElasticsearchTestSupport.java | 64 +++-------------------
.../common/ElasticSearchProperties.java | 3 +
.../ElasticSearchLocalContainerService.java | 58 ++++++++++++++++++--
.../services/ElasticSearchService.java | 12 ++++
.../services/ElasticSearchServiceFactory.java | 62 +++++++++++++++++++++
.../services/RemoteElasticSearchService.java | 24 ++++++++
8 files changed, 166 insertions(+), 65 deletions(-)
diff --git a/components/camel-elasticsearch/src/test/java/org/apache/camel/component/es/integration/ElasticsearchScrollSearchIT.java b/components/camel-elasticsearch/src/test/java/org/apache/camel/component/es/integration/ElasticsearchScrollSearchIT.java
index 0a3bb397013..32b864d13f0 100644
--- a/components/camel-elasticsearch/src/test/java/org/apache/camel/component/es/integration/ElasticsearchScrollSearchIT.java
+++ b/components/camel-elasticsearch/src/test/java/org/apache/camel/component/es/integration/ElasticsearchScrollSearchIT.java
@@ -46,7 +46,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
class ElasticsearchScrollSearchIT extends ElasticsearchTestSupport {
- private static final String TWITTER_ES_INDEX_NAME = "twitter";
+ private static final String TWITTER_ES_INDEX_NAME = "scroll-search";
private static final String SPLIT_TWITTER_ES_INDEX_NAME = "split-" + TWITTER_ES_INDEX_NAME;
@Test
diff --git a/components/camel-elasticsearch/src/test/java/org/apache/camel/component/es/integration/ElasticsearchSizeLimitIT.java b/components/camel-elasticsearch/src/test/java/org/apache/camel/component/es/integration/ElasticsearchSizeLimitIT.java
index 560d68acd5a..fc0fe0d07b0 100644
--- a/components/camel-elasticsearch/src/test/java/org/apache/camel/component/es/integration/ElasticsearchSizeLimitIT.java
+++ b/components/camel-elasticsearch/src/test/java/org/apache/camel/component/es/integration/ElasticsearchSizeLimitIT.java
@@ -51,11 +51,11 @@ class ElasticsearchSizeLimitIT extends ElasticsearchTestSupport {
@Override
public void configure() {
from("direct:index")
- .to("elasticsearch://elasticsearch?operation=Index&indexName=twitter");
+ .to("elasticsearch://elasticsearch?operation=Index&indexName=size-limit");
from("direct:searchWithSizeTwo")
- .to("elasticsearch://elasticsearch?operation=Search&indexName=twitter&size=2");
+ .to("elasticsearch://elasticsearch?operation=Search&indexName=size-limit&size=2");
from("direct:searchFrom3")
- .to("elasticsearch://elasticsearch?operation=Search&indexName=twitter&from=3");
+ .to("elasticsearch://elasticsearch?operation=Search&indexName=size-limit&from=3");
}
};
}
diff --git a/components/camel-elasticsearch/src/test/java/org/apache/camel/component/es/integration/ElasticsearchTestSupport.java b/components/camel-elasticsearch/src/test/java/org/apache/camel/component/es/integration/ElasticsearchTestSupport.java
index 5d8377a6348..fe354fc579d 100644
--- a/components/camel-elasticsearch/src/test/java/org/apache/camel/component/es/integration/ElasticsearchTestSupport.java
+++ b/components/camel-elasticsearch/src/test/java/org/apache/camel/component/es/integration/ElasticsearchTestSupport.java
@@ -16,21 +16,14 @@
*/
package org.apache.camel.component.es.integration;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.time.Duration;
import java.util.HashMap;
import java.util.Map;
-import javax.net.ssl.SSLContext;
-
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.json.jackson.JacksonJsonpMapper;
import co.elastic.clients.transport.rest_client.RestClientTransport;
import org.apache.camel.CamelContext;
import org.apache.camel.component.es.ElasticsearchComponent;
-import org.apache.camel.test.infra.elasticsearch.services.ElasticSearchLocalContainerService;
import org.apache.camel.test.infra.elasticsearch.services.ElasticSearchService;
import org.apache.camel.test.infra.elasticsearch.services.ElasticSearchServiceFactory;
import org.apache.camel.test.junit5.CamelTestSupport;
@@ -45,62 +38,18 @@ import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
-import org.testcontainers.utility.Base58;
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class ElasticsearchTestSupport extends CamelTestSupport {
- public static final int ELASTICSEARCH_DEFAULT_PORT = 9200;
- public static final int ELASTICSEARCH_DEFAULT_TCP_PORT = 9300;
-
@RegisterExtension
- public static ElasticSearchService service = ElasticSearchServiceFactory
- .builder()
- .addLocalMapping(ElasticsearchTestSupport::createElasticSearchService)
- .build();
+ protected static ElasticSearchService service = ElasticSearchServiceFactory.createSingletonService();
protected static String clusterName = "docker-cluster";
protected static RestClient restClient;
protected static ElasticsearchClient client;
- private static Path certPath;
- private static SSLContext sslContext;
- private static final String USER_NAME = "elastic";
- private static final String PASSWORD = "s3cret";
private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchTestSupport.class);
- private static ElasticSearchLocalContainerService createElasticSearchService() {
- ElasticSearchLocalContainerService ret
- = new ElasticSearchLocalContainerService("docker.elastic.co/elasticsearch/elasticsearch:8.6.2") {
- @Override
- public void registerProperties() {
- super.registerProperties();
- getContainer().caCertAsBytes().ifPresent(content -> {
- try {
- certPath = Files.createTempFile("http_ca", ".crt");
- Files.write(certPath, content);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- });
- sslContext = getContainer().createSslContextFromCa();
- }
- };
-
- ret.getContainer()
- .withNetworkAliases("elasticsearch-" + Base58.randomString(6))
- .withPassword(PASSWORD)
- .withExposedPorts(ELASTICSEARCH_DEFAULT_PORT, ELASTICSEARCH_DEFAULT_TCP_PORT)
- // Increase the timeout from 60 seconds to 90 seconds to ensure that it will be long enough
- // on the build pipeline
- .setWaitStrategy(
- new LogMessageWaitStrategy()
- .withRegEx(".*(\"message\":\\s?\"started[\\s?|\"].*|] started\n$)")
- .withStartupTimeout(Duration.ofSeconds(90)));
-
- return ret;
- }
-
@Override
protected void setupResources() throws Exception {
super.setupResources();
@@ -108,11 +57,12 @@ public class ElasticsearchTestSupport extends CamelTestSupport {
= new HttpHost(service.getElasticSearchHost(), service.getPort(), "https");
final RestClientBuilder builder = RestClient.builder(host);
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
- credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(USER_NAME, PASSWORD));
+ credentialsProvider.setCredentials(AuthScope.ANY,
+ new UsernamePasswordCredentials(service.getUsername(), service.getPassword()));
builder.setHttpClientConfigCallback(
httpClientBuilder -> {
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
- httpClientBuilder.setSSLContext(sslContext);
+ httpClientBuilder.setSSLContext(service.getSslContext().orElseThrow());
return httpClientBuilder;
});
restClient = builder.build();
@@ -132,9 +82,9 @@ public class ElasticsearchTestSupport extends CamelTestSupport {
final ElasticsearchComponent elasticsearchComponent = new ElasticsearchComponent();
elasticsearchComponent.setEnableSSL(true);
elasticsearchComponent.setHostAddresses(service.getHttpHostAddress());
- elasticsearchComponent.setUser(USER_NAME);
- elasticsearchComponent.setPassword(PASSWORD);
- elasticsearchComponent.setCertificatePath("file:" + certPath.toString());
+ elasticsearchComponent.setUser(service.getUsername());
+ elasticsearchComponent.setPassword(service.getPassword());
+ elasticsearchComponent.setCertificatePath("file:%s".formatted(service.getCertificatePath().orElseThrow()));
CamelContext context = super.createCamelContext();
context.addComponent("elasticsearch", elasticsearchComponent);
diff --git a/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/common/ElasticSearchProperties.java b/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/common/ElasticSearchProperties.java
index 07a7bda5090..3e9ed756d3a 100644
--- a/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/common/ElasticSearchProperties.java
+++ b/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/common/ElasticSearchProperties.java
@@ -22,6 +22,9 @@ import org.apache.camel.test.infra.common.services.ContainerEnvironmentUtil;
public final class ElasticSearchProperties {
public static final String ELASTIC_SEARCH_HOST = "elasticsearch.host";
public static final String ELASTIC_SEARCH_PORT = "elasticsearch.port";
+ public static final String ELASTIC_SEARCH_CERTIFICATE_PATH = "elasticsearch.certificate.path";
+ public static final String ELASTIC_SEARCH_USERNAME = "elasticsearch.username";
+ public static final String ELASTIC_SEARCH_PASSWORD = "elasticsearch.password";
public static final String ELASTIC_SEARCH_CONTAINER = "elasticsearch.container";
public static final String ELASTIC_SEARCH_CONTAINER_STARTUP
= ELASTIC_SEARCH_CONTAINER + ContainerEnvironmentUtil.STARTUP_ATTEMPTS_PROPERTY;
diff --git a/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/services/ElasticSearchLocalContainerService.java b/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/services/ElasticSearchLocalContainerService.java
index 6339b7f3d3b..8730351387b 100644
--- a/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/services/ElasticSearchLocalContainerService.java
+++ b/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/services/ElasticSearchLocalContainerService.java
@@ -17,19 +17,31 @@
package org.apache.camel.test.infra.elasticsearch.services;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.Objects;
+import java.util.Optional;
+
+import javax.net.ssl.SSLContext;
+
import org.apache.camel.test.infra.common.services.ContainerEnvironmentUtil;
import org.apache.camel.test.infra.common.services.ContainerService;
import org.apache.camel.test.infra.elasticsearch.common.ElasticSearchProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
import org.testcontainers.elasticsearch.ElasticsearchContainer;
public class ElasticSearchLocalContainerService implements ElasticSearchService, ContainerService<ElasticsearchContainer> {
- public static final String DEFAULT_ELASTIC_SEARCH_CONTAINER = "docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2";
-
+ public static final String DEFAULT_ELASTIC_SEARCH_CONTAINER = "docker.elastic.co/elasticsearch/elasticsearch:8.6.2";
private static final Logger LOG = LoggerFactory.getLogger(ElasticSearchLocalContainerService.class);
private static final int ELASTIC_SEARCH_PORT = 9200;
-
+ private static final String USER_NAME = "elastic";
+ private static final String PASSWORD = "s3cret";
+ private Path certPath;
+ private SSLContext sslContext;
private final ElasticsearchContainer container;
public ElasticSearchLocalContainerService() {
@@ -45,7 +57,16 @@ public class ElasticSearchLocalContainerService implements ElasticSearchService,
}
protected ElasticsearchContainer initContainer(String imageName) {
- return new ElasticsearchContainer(imageName);
+ ElasticsearchContainer elasticsearchContainer = new ElasticsearchContainer(imageName)
+ .withPassword(PASSWORD);
+ // Increase the timeout from 60 seconds to 90 seconds to ensure that it will be long enough
+ // on the build pipeline
+ elasticsearchContainer.setWaitStrategy(
+ new LogMessageWaitStrategy()
+ .withRegEx(".*(\"message\":\\s?\"started[\\s?|\"].*|] started\n$)")
+ .withStartupTimeout(Duration.ofSeconds(90)));
+ return elasticsearchContainer;
+
}
@Override
@@ -67,6 +88,15 @@ public class ElasticSearchLocalContainerService implements ElasticSearchService,
public void registerProperties() {
System.setProperty(ElasticSearchProperties.ELASTIC_SEARCH_HOST, getElasticSearchHost());
System.setProperty(ElasticSearchProperties.ELASTIC_SEARCH_PORT, String.valueOf(getPort()));
+ getContainer().caCertAsBytes().ifPresent(content -> {
+ try {
+ certPath = Files.createTempFile("http_ca", ".crt");
+ Files.write(certPath, content);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ sslContext = getContainer().createSslContextFromCa();
}
@Override
@@ -91,4 +121,24 @@ public class ElasticSearchLocalContainerService implements ElasticSearchService,
public ElasticsearchContainer getContainer() {
return container;
}
+
+ @Override
+ public Optional<String> getCertificatePath() {
+ return Optional.ofNullable(certPath).map(Objects::toString);
+ }
+
+ @Override
+ public Optional<SSLContext> getSslContext() {
+ return Optional.ofNullable(sslContext);
+ }
+
+ @Override
+ public String getUsername() {
+ return USER_NAME;
+ }
+
+ @Override
+ public String getPassword() {
+ return PASSWORD;
+ }
}
diff --git a/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/services/ElasticSearchService.java b/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/services/ElasticSearchService.java
index 05dff9e4521..d28067cac73 100644
--- a/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/services/ElasticSearchService.java
+++ b/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/services/ElasticSearchService.java
@@ -17,6 +17,10 @@
package org.apache.camel.test.infra.elasticsearch.services;
+import java.util.Optional;
+
+import javax.net.ssl.SSLContext;
+
import org.apache.camel.test.infra.common.services.TestService;
public interface ElasticSearchService extends TestService {
@@ -28,4 +32,12 @@ public interface ElasticSearchService extends TestService {
default String getHttpHostAddress() {
return String.format("%s:%d", getElasticSearchHost(), getPort());
}
+
+ Optional<String> getCertificatePath();
+
+ Optional<SSLContext> getSslContext();
+
+ String getUsername();
+
+ String getPassword();
}
diff --git a/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/services/ElasticSearchServiceFactory.java b/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/services/ElasticSearchServiceFactory.java
index 4311ff38d3b..8198e320c35 100644
--- a/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/services/ElasticSearchServiceFactory.java
+++ b/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/services/ElasticSearchServiceFactory.java
@@ -17,9 +17,56 @@
package org.apache.camel.test.infra.elasticsearch.services;
+import java.util.Optional;
+
+import javax.net.ssl.SSLContext;
+
import org.apache.camel.test.infra.common.services.SimpleTestServiceBuilder;
+import org.apache.camel.test.infra.common.services.SingletonService;
public final class ElasticSearchServiceFactory {
+
+ static class SingletonElasticSearchService extends SingletonService<ElasticSearchService> implements ElasticSearchService {
+ public SingletonElasticSearchService(ElasticSearchService service, String name) {
+ super(service, name);
+ }
+
+ @Override
+ public int getPort() {
+ return getService().getPort();
+ }
+
+ @Override
+ public String getElasticSearchHost() {
+ return getService().getElasticSearchHost();
+ }
+
+ @Override
+ public String getHttpHostAddress() {
+ return getService().getHttpHostAddress();
+ }
+
+ @Override
+ public Optional<String> getCertificatePath() {
+ return getService().getCertificatePath();
+ }
+
+ @Override
+ public Optional<SSLContext> getSslContext() {
+ return getService().getSslContext();
+ }
+
+ @Override
+ public String getUsername() {
+ return getService().getUsername();
+ }
+
+ @Override
+ public String getPassword() {
+ return getService().getPassword();
+ }
+ }
+
private ElasticSearchServiceFactory() {
}
@@ -34,4 +81,19 @@ public final class ElasticSearchServiceFactory {
.addRemoteMapping(RemoteElasticSearchService::new)
.build();
}
+
+ public static ElasticSearchService createSingletonService() {
+ return SingletonServiceHolder.INSTANCE;
+ }
+
+ private static class SingletonServiceHolder {
+ static final ElasticSearchService INSTANCE;
+ static {
+ SimpleTestServiceBuilder<ElasticSearchService> instance = builder();
+ instance.addLocalMapping(
+ () -> new SingletonElasticSearchService(new ElasticSearchLocalContainerService(), "elastic"))
+ .addRemoteMapping(RemoteElasticSearchService::new);
+ INSTANCE = instance.build();
+ }
+ }
}
diff --git a/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/services/RemoteElasticSearchService.java b/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/services/RemoteElasticSearchService.java
index af4b29f7ca8..f23d9def19e 100644
--- a/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/services/RemoteElasticSearchService.java
+++ b/test-infra/camel-test-infra-elasticsearch/src/test/java/org/apache/camel/test/infra/elasticsearch/services/RemoteElasticSearchService.java
@@ -17,6 +17,10 @@
package org.apache.camel.test.infra.elasticsearch.services;
+import java.util.Optional;
+
+import javax.net.ssl.SSLContext;
+
import org.apache.camel.test.infra.elasticsearch.common.ElasticSearchProperties;
public class RemoteElasticSearchService implements ElasticSearchService {
@@ -52,4 +56,24 @@ public class RemoteElasticSearchService implements ElasticSearchService {
public void shutdown() {
// NO-OP
}
+
+ @Override
+ public Optional<String> getCertificatePath() {
+ return Optional.ofNullable(System.getProperty(ElasticSearchProperties.ELASTIC_SEARCH_CERTIFICATE_PATH));
+ }
+
+ @Override
+ public Optional<SSLContext> getSslContext() {
+ return Optional.empty();
+ }
+
+ @Override
+ public String getUsername() {
+ return System.getProperty(ElasticSearchProperties.ELASTIC_SEARCH_USERNAME);
+ }
+
+ @Override
+ public String getPassword() {
+ return System.getProperty(ElasticSearchProperties.ELASTIC_SEARCH_PASSWORD);
+ }
}