You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by ro...@apache.org on 2019/11/14 09:13:26 UTC
[james-project] 06/07: JAMES-2905 ElasticSearch configuration
various improvements
This is an automated email from the ASF dual-hosted git repository.
rouazana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 8b46a4b68145334927655b8e10800be0aeb8ab0d
Author: Tran Tien Duc <dt...@linagora.com>
AuthorDate: Wed Nov 13 10:21:40 2019 +0700
JAMES-2905 ElasticSearch configuration various improvements
- Using a written HostNameVerifier
- Using File + char [] in TrustStore
- Correct documentations
...
---
.../apache/james/backends/es/ClientProvider.java | 10 ++--
.../backends/es/ElasticSearchConfiguration.java | 53 +++++++++++++---------
.../james/backends/es/DockerElasticSearch.java | 4 +-
.../backends/es/ElasticSearchClusterExtension.java | 4 +-
.../es/ElasticSearchConfigurationTest.java | 32 +++++++++++--
.../src/test/resources/auth-es/README.md | 2 +-
.../destination/conf/elasticsearch.properties | 5 +-
.../destination/conf/elasticsearch.properties | 2 +-
.../destination/conf/elasticsearch.properties | 5 +-
.../destination/conf/elasticsearch.properties | 2 +-
src/site/xdoc/server/config-elasticsearch.xml | 23 ++++++----
11 files changed, 94 insertions(+), 48 deletions(-)
diff --git a/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ClientProvider.java b/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ClientProvider.java
index 923a820..50a51e7 100644
--- a/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ClientProvider.java
+++ b/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ClientProvider.java
@@ -18,7 +18,6 @@
****************************************************************/
package org.apache.james.backends.es;
-import java.io.File;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
@@ -30,6 +29,7 @@ import java.time.LocalDateTime;
import javax.annotation.PreDestroy;
import javax.inject.Inject;
import javax.inject.Provider;
+import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
import org.apache.commons.lang3.NotImplementedException;
@@ -38,7 +38,6 @@ import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.apache.http.ssl.SSLContextBuilder;
@@ -61,6 +60,7 @@ public class ClientProvider implements Provider<RestHighLevelClient> {
private static class HttpAsyncClientConfigurer {
private static final TrustStrategy TRUST_ALL = (x509Certificates, authType) -> true;
+ private static final HostnameVerifier ACCEPT_ANY_HOST = (hostname, sslSession) -> true;
private final ElasticSearchConfiguration configuration;
@@ -94,7 +94,7 @@ public class ClientProvider implements Provider<RestHighLevelClient> {
try {
builder
.setSSLContext(sslContext())
- .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE);
+ .setSSLHostnameVerifier(ACCEPT_ANY_HOST);
} catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException | CertificateException | IOException e) {
throw new RuntimeException("Cannot set SSL options to the builder", e);
}
@@ -131,7 +131,7 @@ public class ClientProvider implements Provider<RestHighLevelClient> {
.orElseThrow(() -> new IllegalStateException("SSLTrustStore cannot to be empty"));
return sslContextBuilder
- .loadTrustMaterial(new File(trustStore.getFilePath()), trustStore.getPassword().toCharArray());
+ .loadTrustMaterial(trustStore.getFile(), trustStore.getPassword());
}
private void configureAuthentication(HttpAsyncClientBuilder builder) {
@@ -139,7 +139,7 @@ public class ClientProvider implements Provider<RestHighLevelClient> {
.ifPresent(credential -> {
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
- new UsernamePasswordCredentials(credential.getUsername(), credential.getPassword()));
+ new UsernamePasswordCredentials(credential.getUsername(), String.valueOf(credential.getPassword())));
builder.setDefaultCredentialsProvider(credentialsProvider);
});
}
diff --git a/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ElasticSearchConfiguration.java b/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ElasticSearchConfiguration.java
index 82ce0d6..47681a5 100644
--- a/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ElasticSearchConfiguration.java
+++ b/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ElasticSearchConfiguration.java
@@ -21,10 +21,14 @@ package org.apache.james.backends.es;
import static org.apache.james.backends.es.ElasticSearchConfiguration.SSLTrustConfiguration.SSLValidationStrategy.OVERRIDE;
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
+import java.util.Locale;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Stream;
@@ -47,8 +51,11 @@ public class ElasticSearchConfiguration {
HTTPS("https");
public static HostScheme of(String schemeValue) {
+ Preconditions.checkNotNull(schemeValue);
+
return Arrays.stream(values())
- .filter(hostScheme -> hostScheme.value.equalsIgnoreCase(schemeValue))
+ .filter(hostScheme -> hostScheme.value.toLowerCase(Locale.US)
+ .equals(schemeValue.toLowerCase(Locale.US)))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException(
String.format("Unknown HostScheme '%s'", schemeValue)));
@@ -68,21 +75,21 @@ public class ElasticSearchConfiguration {
}
private final String username;
- private final String password;
+ private final char[] password;
private Credential(String username, String password) {
Preconditions.checkNotNull(username, "username cannot be null when password is specified");
Preconditions.checkNotNull(password, "password cannot be null when username is specified");
this.username = username;
- this.password = password;
+ this.password = password.toCharArray();
}
public String getUsername() {
return username;
}
- public String getPassword() {
+ public char[] getPassword() {
return password;
}
@@ -92,14 +99,14 @@ public class ElasticSearchConfiguration {
Credential that = (Credential) o;
return Objects.equals(this.username, that.username)
- && Objects.equals(this.password, that.password);
+ && Arrays.equals(this.password, that.password);
}
return false;
}
@Override
public final int hashCode() {
- return Objects.hash(username, password);
+ return Objects.hash(username, Arrays.hashCode(password));
}
}
@@ -127,22 +134,26 @@ public class ElasticSearchConfiguration {
return new SSLTrustStore(filePath, password);
}
- private final String filePath;
- private final String password;
+ private final File file;
+ private final char[] password;
private SSLTrustStore(String filePath, String password) {
- Preconditions.checkNotNull(filePath, ELASTICSEARCH_HTTPS_TRUST_STORE_PATH + " cannot be null when " + ELASTICSEARCH_HTTPS_TRUST_STORE_PASSWORD + " is specified");
- Preconditions.checkNotNull(password, ELASTICSEARCH_HTTPS_TRUST_STORE_PASSWORD + " cannot be null when " + ELASTICSEARCH_HTTPS_TRUST_STORE_PATH + " is specified");
-
- this.filePath = filePath;
- this.password = password;
+ Preconditions.checkNotNull(filePath,
+ ELASTICSEARCH_HTTPS_TRUST_STORE_PATH + " cannot be null when " + ELASTICSEARCH_HTTPS_TRUST_STORE_PASSWORD + " is specified");
+ Preconditions.checkNotNull(password,
+ ELASTICSEARCH_HTTPS_TRUST_STORE_PASSWORD + " cannot be null when " + ELASTICSEARCH_HTTPS_TRUST_STORE_PATH + " is specified");
+ Preconditions.checkArgument(Files.exists(Paths.get(filePath)),
+ String.format("the file '%s' from property '%s' doesn't exist", filePath, ELASTICSEARCH_HTTPS_TRUST_STORE_PATH));
+
+ this.file = new File(filePath);
+ this.password = password.toCharArray();
}
- public String getFilePath() {
- return filePath;
+ public File getFile() {
+ return file;
}
- public String getPassword() {
+ public char[] getPassword() {
return password;
}
@@ -151,15 +162,15 @@ public class ElasticSearchConfiguration {
if (o instanceof SSLTrustStore) {
SSLTrustStore that = (SSLTrustStore) o;
- return Objects.equals(this.filePath, that.filePath)
- && Objects.equals(this.password, that.password);
+ return Objects.equals(this.file, that.file)
+ && Arrays.equals(this.password, that.password);
}
return false;
}
@Override
public final int hashCode() {
- return Objects.hash(filePath, password);
+ return Objects.hash(file, Arrays.hashCode(password));
}
}
@@ -181,7 +192,7 @@ public class ElasticSearchConfiguration {
private SSLTrustConfiguration(SSLValidationStrategy strategy, Optional<SSLTrustStore> trustStore) {
Preconditions.checkNotNull(strategy);
Preconditions.checkNotNull(trustStore);
- Preconditions.checkArgument(strategy != OVERRIDE || trustStore.isPresent(), "OVERRIDE strategy requires trustStore to be present");
+ Preconditions.checkArgument(strategy != OVERRIDE || trustStore.isPresent(), OVERRIDE.name() + " strategy requires trustStore to be present");
this.strategy = strategy;
this.trustStore = trustStore;
@@ -292,7 +303,7 @@ public class ElasticSearchConfiguration {
}
public Builder sslTrustConfiguration(SSLTrustConfiguration sslTrustConfiguration) {
- this.sslTrustConfiguration = Optional.ofNullable(sslTrustConfiguration);
+ this.sslTrustConfiguration = Optional.of(sslTrustConfiguration);
return this;
}
diff --git a/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/DockerElasticSearch.java b/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/DockerElasticSearch.java
index 2424477..03fda7f 100644
--- a/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/DockerElasticSearch.java
+++ b/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/DockerElasticSearch.java
@@ -62,7 +62,7 @@ public interface DockerElasticSearch {
interface ElasticSearchAPI {
class Builder {
- private static final HostnameVerifier ACCEPT_ANY_HOST = (hostname1, sslSession) -> true;
+ private static final HostnameVerifier ACCEPT_ANY_HOST = (hostname, sslSession) -> true;
private static final TrustManager[] TRUST_ALL = new TrustManager[] {
new X509TrustManager() {
@@ -88,7 +88,7 @@ public interface DockerElasticSearch {
public Builder credential(Credential credential) {
requestBuilder.requestInterceptor(
- new BasicAuthRequestInterceptor(credential.getUsername(), credential.getPassword()));
+ new BasicAuthRequestInterceptor(credential.getUsername(), String.valueOf(credential.getPassword())));
return this;
}
diff --git a/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ElasticSearchClusterExtension.java b/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ElasticSearchClusterExtension.java
index 3e73fe0..41660b3 100644
--- a/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ElasticSearchClusterExtension.java
+++ b/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ElasticSearchClusterExtension.java
@@ -58,8 +58,8 @@ class ElasticSearchClusterExtension implements AfterAllCallback, BeforeAllCallba
es1.cleanUpData();
}},
() -> {
- if (es1.isRunning()) {
- es1.cleanUpData();
+ if (es2.isRunning()) {
+ es2.cleanUpData();
}});
}
diff --git a/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ElasticSearchConfigurationTest.java b/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ElasticSearchConfigurationTest.java
index 6cb1867..cf50942 100644
--- a/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ElasticSearchConfigurationTest.java
+++ b/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ElasticSearchConfigurationTest.java
@@ -159,7 +159,7 @@ class ElasticSearchConfigurationTest {
assertThatThrownBy(() -> ElasticSearchConfiguration.fromProperties(configuration))
.isInstanceOf(NullPointerException.class)
- .hasMessage("password cannot be null when filePath is specified");
+ .hasMessage("elasticsearch.hostScheme.https.trustStorePassword cannot be null when elasticsearch.hostScheme.https.trustStorePath is specified");
}
@Test
@@ -172,7 +172,33 @@ class ElasticSearchConfigurationTest {
assertThatThrownBy(() -> ElasticSearchConfiguration.fromProperties(configuration))
.isInstanceOf(NullPointerException.class)
- .hasMessage("filePath cannot be null when password is specified");
+ .hasMessage("elasticsearch.hostScheme.https.trustStorePath cannot be null when elasticsearch.hostScheme.https.trustStorePassword is specified");
+ }
+
+ @Test
+ void fromPropertiesShouldThrowWhenTrustStoreIsNotProvided() throws Exception {
+ PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.addProperty("elasticsearch.hosts", "127.0.0.1");
+
+ configuration.addProperty("elasticsearch.hostScheme.https.sslValidationStrategy", "override");
+
+ assertThatThrownBy(() -> ElasticSearchConfiguration.fromProperties(configuration))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("OVERRIDE strategy requires trustStore to be present");
+ }
+
+ @Test
+ void fromPropertiesShouldThrowWhenTrustStorePathDoesntExist() throws Exception {
+ PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.addProperty("elasticsearch.hosts", "127.0.0.1");
+
+ configuration.addProperty("elasticsearch.hostScheme.https.sslValidationStrategy", "override");
+ configuration.addProperty("elasticsearch.hostScheme.https.trustStorePath", "/home/james/ServerTrustStore.jks");
+ configuration.addProperty("elasticsearch.hostScheme.https.trustStorePassword", "password");
+
+ assertThatThrownBy(() -> ElasticSearchConfiguration.fromProperties(configuration))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("the file '/home/james/ServerTrustStore.jks' from property 'elasticsearch.hostScheme.https.trustStorePath' doesn't exist");
}
@Test
@@ -181,7 +207,7 @@ class ElasticSearchConfigurationTest {
configuration.addProperty("elasticsearch.hosts", "127.0.0.1");
String strategy = "override";
- String trustStorePath = "/home/james/ServerTrustStore.jks";
+ String trustStorePath = "src/test/resources/auth-es/server.jks";
String trustStorePassword = "secret";
configuration.addProperty("elasticsearch.hostScheme.https.sslValidationStrategy", strategy);
diff --git a/backends-common/elasticsearch/src/test/resources/auth-es/README.md b/backends-common/elasticsearch/src/test/resources/auth-es/README.md
index b7d945e..5dd42de 100644
--- a/backends-common/elasticsearch/src/test/resources/auth-es/README.md
+++ b/backends-common/elasticsearch/src/test/resources/auth-es/README.md
@@ -4,7 +4,7 @@
Contains nginx configuration files:
- reverse_elasticsearch.conf: allow nginx to be the proxy connecting to ElasticSearch
- - passwd: Nginx credentials file store, each record follow the format: `username:encrypted-password`
+ - passwd: Nginx credentials file store, each record follows the format: `username:encrypted-password`
### default.crt & default.key
diff --git a/dockerfiles/run/guice/cassandra-ldap/destination/conf/elasticsearch.properties b/dockerfiles/run/guice/cassandra-ldap/destination/conf/elasticsearch.properties
index f28c38a..b2eefc5 100644
--- a/dockerfiles/run/guice/cassandra-ldap/destination/conf/elasticsearch.properties
+++ b/dockerfiles/run/guice/cassandra-ldap/destination/conf/elasticsearch.properties
@@ -25,11 +25,14 @@
elasticsearch.masterHost=elasticsearch
elasticsearch.port=9200
+# Optional. Only http or https are accepted, default is http
+# elasticsearch.hostScheme=http
+
# Optional, default is `default`
# Choosing the SSL check strategy when using https scheme
# default: Use the default SSL TrustStore of the system.
# ignore: Ignore SSL Validation check (not recommended).
-# override: Override the SSL Context to use a custome TrustStore containing ES server's certificate.
+# override: Override the SSL Context to use a custom TrustStore containing ES server's certificate.
# elasticsearch.hostScheme.https.sslValidationStrategy=default
# Optional. Required when using 'https' scheme and 'override' sslValidationStrategy
diff --git a/dockerfiles/run/guice/cassandra-rabbitmq-ldap/destination/conf/elasticsearch.properties b/dockerfiles/run/guice/cassandra-rabbitmq-ldap/destination/conf/elasticsearch.properties
index 3490e61..eecf449 100644
--- a/dockerfiles/run/guice/cassandra-rabbitmq-ldap/destination/conf/elasticsearch.properties
+++ b/dockerfiles/run/guice/cassandra-rabbitmq-ldap/destination/conf/elasticsearch.properties
@@ -31,7 +31,7 @@ elasticsearch.port=9200
# Choosing the SSL check strategy when using https scheme
# default: Use the default SSL TrustStore of the system.
# ignore: Ignore SSL Validation check (not recommended).
-# override: Override the SSL Context to use a custome TrustStore containing ES server's certificate.
+# override: Override the SSL Context to use a custom TrustStore containing ES server's certificate.
# elasticsearch.hostScheme.https.sslValidationStrategy=default
# Optional. Required when using 'https' scheme and 'override' sslValidationStrategy
diff --git a/dockerfiles/run/guice/cassandra-rabbitmq/destination/conf/elasticsearch.properties b/dockerfiles/run/guice/cassandra-rabbitmq/destination/conf/elasticsearch.properties
index 7c23c72..eecf449 100644
--- a/dockerfiles/run/guice/cassandra-rabbitmq/destination/conf/elasticsearch.properties
+++ b/dockerfiles/run/guice/cassandra-rabbitmq/destination/conf/elasticsearch.properties
@@ -24,11 +24,14 @@
elasticsearch.masterHost=elasticsearch
elasticsearch.port=9200
+# Optional. Only http or https are accepted, default is http
+# elasticsearch.hostScheme=http
+
# Optional, default is `default`
# Choosing the SSL check strategy when using https scheme
# default: Use the default SSL TrustStore of the system.
# ignore: Ignore SSL Validation check (not recommended).
-# override: Override the SSL Context to use a custome TrustStore containing ES server's certificate.
+# override: Override the SSL Context to use a custom TrustStore containing ES server's certificate.
# elasticsearch.hostScheme.https.sslValidationStrategy=default
# Optional. Required when using 'https' scheme and 'override' sslValidationStrategy
diff --git a/dockerfiles/run/guice/cassandra/destination/conf/elasticsearch.properties b/dockerfiles/run/guice/cassandra/destination/conf/elasticsearch.properties
index 077e76c..b2eefc5 100644
--- a/dockerfiles/run/guice/cassandra/destination/conf/elasticsearch.properties
+++ b/dockerfiles/run/guice/cassandra/destination/conf/elasticsearch.properties
@@ -32,7 +32,7 @@ elasticsearch.port=9200
# Choosing the SSL check strategy when using https scheme
# default: Use the default SSL TrustStore of the system.
# ignore: Ignore SSL Validation check (not recommended).
-# override: Override the SSL Context to use a custome TrustStore containing ES server's certificate.
+# override: Override the SSL Context to use a custom TrustStore containing ES server's certificate.
# elasticsearch.hostScheme.https.sslValidationStrategy=default
# Optional. Required when using 'https' scheme and 'override' sslValidationStrategy
diff --git a/src/site/xdoc/server/config-elasticsearch.xml b/src/site/xdoc/server/config-elasticsearch.xml
index 77f0fc4..f763272 100644
--- a/src/site/xdoc/server/config-elasticsearch.xml
+++ b/src/site/xdoc/server/config-elasticsearch.xml
@@ -205,21 +205,24 @@
<dd>
default: Use the default SSL TrustStore of the system.
ignore: Ignore SSL Validation check (not recommended).
- override: Override the SSL Context to use a custome TrustStore containing ES server's certificate.
+ override: Override the SSL Context to use a custom TrustStore containing ES server's certificate.
</dd>
</dl>
<p>
- In some cases, you want to secure ES to protect it from unauthorized requests,
- assuming with the ES is using <strong>https</strong> with a self signed certificate.
- Which means you should configure the ES RestHighLevelClient to trust your self signed certificate.
-
- There are two ways on client side: ignoring SSL check or configure to trust the server's certificate.
+ In some cases, you want to secure the connection from clients to ES by setting up a <strong>https</strong> protocol
+ with a self signed certificate. And you prefer to left the system ca-certificates un touch.
+ There are possible solutions to let the ES RestHighLevelClient to trust your self signed certificate.
+ </p>
+ <p>
+ First solution: ignoring SSL check.
In case you want to ignore the SSL check, simply, just don't specify below options. Otherwise, configuring the trust
requires some prerequisites and they are explained in below block.
-
+ </p>
+ <p>
+ Second solution: importing a TrustStore containing the certificate into SSL context.
A certificate normally contains two parts: a public part in .crt file, another private part in .key file.
- To trust the server, the client need to be acknowledged that the server's certificate is in the list of
+ To trust the server, the client needs to be acknowledged that the server's certificate is in the list of
client's TrustStore. Basically, you can create a local TrustStore file containing the public part of a remote server
by execute this command:
</p>
@@ -239,14 +242,14 @@
<dd>
Optional. Use it when https is configured in elasticsearch.hostScheme, and sslValidationStrategy is <strong>override</strong>
Configure Elasticsearch rest client to use this trustStore file to recognize nginx's ssl certificate.
- Once, you chose <strong>override</strong>, you need to specify both trustStorePath and trustStorePassword.
+ Once you chose <strong>override</strong>, you need to specify both trustStorePath and trustStorePassword.
</dd>
<dt><strong>elasticsearch.hostScheme.https.trustStorePassword</strong></dt>
<dd>
Optional. Use it when https is configured in elasticsearch.hostScheme, and sslValidationStrategy is <strong>override</strong>
Configure Elasticsearch rest client to use this trustStore file with the specified password.
- Once, you chose <strong>override</strong>, you need to specify both trustStorePath and trustStorePassword.
+ Once you chose <strong>override</strong>, you need to specify both trustStorePath and trustStorePassword.
</dd>
</dl>
</section>
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org