You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2020/03/11 08:43:41 UTC
[skywalking] 02/02: Support secretsManagementFile file.
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch vault-support
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit b8127f8c8748d977e4a9714b537a43850190f47b
Author: Wu Sheng <wu...@foxmail.com>
AuthorDate: Wed Mar 11 16:43:20 2020 +0800
Support secretsManagementFile file.
---
dist-material/application.yml | 2 ++
docs/en/setup/backend/backend-storage.md | 14 +++++++++++
.../src/main/resources/application.yml | 2 ++
.../client/elasticsearch/ElasticSearchClient.java | 7 ++++++
.../StorageModuleElasticsearchProvider.java | 16 +++++++-----
.../StorageModuleElasticsearch7Provider.java | 27 ++++++++++++++++++++
.../client/ElasticSearch7Client.java | 29 ++++++++++++++--------
7 files changed, 80 insertions(+), 17 deletions(-)
diff --git a/dist-material/application.yml b/dist-material/application.yml
index 4067fef..8defe81 100644
--- a/dist-material/application.yml
+++ b/dist-material/application.yml
@@ -90,6 +90,7 @@ storage:
# dayStep: ${SW_STORAGE_DAY_STEP:1} # Represent the number of days in the one minute/hour/day index.
# user: ${SW_ES_USER:""}
# password: ${SW_ES_PASSWORD:""}
+# secretsManagementFile: ${SW_ES_SECRETS_MANAGEMENT_FILE:""} # Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool.
# indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:2}
# indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0}
# # Those data TTL settings will override the same settings in core module.
@@ -114,6 +115,7 @@ storage:
# dayStep: ${SW_STORAGE_DAY_STEP:1} # Represent the number of days in the one minute/hour/day index.
# user: ${SW_ES_USER:""}
# password: ${SW_ES_PASSWORD:""}
+# secretsManagementFile: ${SW_ES_SECRETS_MANAGEMENT_FILE:""} # Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool.
# indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:2}
# indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0}
# # Those data TTL settings will override the same settings in core module.
diff --git a/docs/en/setup/backend/backend-storage.md b/docs/en/setup/backend/backend-storage.md
index a9702c9..64f9707 100644
--- a/docs/en/setup/backend/backend-storage.md
+++ b/docs/en/setup/backend/backend-storage.md
@@ -44,6 +44,7 @@ storage:
# nameSpace: ${SW_NAMESPACE:""}
# user: ${SW_ES_USER:""} # User needs to be set when Http Basic authentication is enabled
# password: ${SW_ES_PASSWORD:""} # Password to be set when Http Basic authentication is enabled
+ # secretsManagementFile: ${SW_ES_SECRETS_MANAGEMENT_FILE:""} # Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool.
#trustStorePath: ${SW_SW_STORAGE_ES_SSL_JKS_PATH:""}
#trustStorePass: ${SW_SW_STORAGE_ES_SSL_JKS_PASS:""}
enablePackedDownsampling: ${SW_STORAGE_ENABLE_PACKED_DOWNSAMPLING:true} # Hour and Day metrics will be merged into minute index.
@@ -124,6 +125,19 @@ Such as, if dayStep == 11,
NOTICE, TTL deletion would be affected by these. You should set an extra more dayStep in your TTL. Such as you want to TTL == 30 days and dayStep == 10, you actually need to set TTL = 40;
+### Secrets Management File Of ElasticSearch Username and Password
+The value of `secretsManagementFile` should point to the secrets management file is the file including username and password of ElasticSearch server.
+The file uses the properties format.
+```properties
+user=xxx
+password=yyy
+```
+
+The major difference between using `user/password` configs in the `application.yaml` and this file is, this file is being watched by the OAP server.
+Once it is changed manually or through 3rd party tool, such as [Vault](https://github.com/hashicorp/vault),
+the storage provider will use the new username and password to establish the connection and close the old one. If the information exist in the file,
+the `user/password` will be overrided.
+
### Advanced Configurations For Elasticsearch Index
You can add advanced configurations in `JSON` format to set `ElasticSearch index settings` by following [ElasticSearch doc](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html)
diff --git a/oap-server/server-bootstrap/src/main/resources/application.yml b/oap-server/server-bootstrap/src/main/resources/application.yml
index 3d87b3e..53b5f9b 100755
--- a/oap-server/server-bootstrap/src/main/resources/application.yml
+++ b/oap-server/server-bootstrap/src/main/resources/application.yml
@@ -87,6 +87,7 @@ storage:
# #trustStorePass: ${SW_SW_STORAGE_ES_SSL_JKS_PASS:""}
# user: ${SW_ES_USER:""}
# password: ${SW_ES_PASSWORD:""}
+# secretsManagementFile: ${SW_ES_SECRETS_MANAGEMENT_FILE:""} # Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool.
# enablePackedDownsampling: ${SW_STORAGE_ENABLE_PACKED_DOWNSAMPLING:true} # Hour and Day metrics will be merged into minute index.
# dayStep: ${SW_STORAGE_DAY_STEP:1} # Represent the number of days in the one minute/hour/day index.
# indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:2}
@@ -114,6 +115,7 @@ storage:
dayStep: ${SW_STORAGE_DAY_STEP:1} # Represent the number of days in the one minute/hour/day index.
user: ${SW_ES_USER:""}
password: ${SW_ES_PASSWORD:""}
+ secretsManagementFile: ${SW_ES_SECRETS_MANAGEMENT_FILE:""} # Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool.
indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:2}
indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0}
# Those data TTL settings will override the same settings in core module.
diff --git a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java
index 0f47abe..7343339 100644
--- a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java
+++ b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java
@@ -119,6 +119,13 @@ public class ElasticSearchClient implements Client {
@Override
public void connect() throws IOException, KeyStoreException, NoSuchAlgorithmException, KeyManagementException, CertificateException {
List<HttpHost> hosts = parseClusterNodes(protocol, clusterNodes);
+ if (client != null) {
+ try {
+ client.close();
+ } catch (Throwable t) {
+ log.error("ElasticSearch client reconnection fails based on new config", t);
+ }
+ }
client = createClient(hosts);
client.ping();
}
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java
index eb7e454..b5e68ac 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java
@@ -123,18 +123,22 @@ public class StorageModuleElasticsearchProvider extends ModuleProvider {
if (!StringUtil.isEmpty(config.getSecretsManagementFile())) {
MultipleFilesChangeMonitor monitor = new MultipleFilesChangeMonitor(
- 10, new MultipleFilesChangeMonitor.FilesChangedNotifier() {
- @Override
- public void filesChanged(final List<byte[]> readableContents) throws IOException {
+ 10, readableContents -> {
final byte[] secretsFileContent = readableContents.get(0);
if (secretsFileContent == null) {
return;
}
Properties userAndPass = new Properties();
userAndPass.load(new ByteArrayInputStream(secretsFileContent));
- userAndPass.getProperty("username")
- }
- }, config.getSecretsManagementFile());
+ config.setUser(userAndPass.getProperty("user"));
+ config.setPassword(userAndPass.getProperty("password"));
+
+ if (elasticSearchClient == null) {
+ //In the startup process, we just need to change the username/password
+ } else {
+ elasticSearchClient.connect();
+ }
+ }, config.getSecretsManagementFile());
/**
* By leveraging the sync update check feature when startup.
*/
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/StorageModuleElasticsearch7Provider.java b/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/StorageModuleElasticsearch7Provider.java
index 82f996a..fe4a854 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/StorageModuleElasticsearch7Provider.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/StorageModuleElasticsearch7Provider.java
@@ -18,11 +18,13 @@
package org.apache.skywalking.oap.server.storage.plugin.elasticsearch7;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
+import java.util.Properties;
import org.apache.skywalking.apm.util.StringUtil;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.config.ConfigService;
@@ -52,6 +54,7 @@ import org.apache.skywalking.oap.server.library.module.ModuleDefine;
import org.apache.skywalking.oap.server.library.module.ModuleProvider;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException;
+import org.apache.skywalking.oap.server.library.util.MultipleFilesChangeMonitor;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.BatchProcessEsDAO;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.HistoryDeleteEsDAO;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ProfileTaskLogEsDAO;
@@ -108,6 +111,30 @@ public class StorageModuleElasticsearch7Provider extends ModuleProvider {
if (!StringUtil.isEmpty(config.getNameSpace())) {
config.setNameSpace(config.getNameSpace().toLowerCase());
}
+ if (!StringUtil.isEmpty(config.getSecretsManagementFile())) {
+ MultipleFilesChangeMonitor monitor = new MultipleFilesChangeMonitor(
+ 10, readableContents -> {
+ final byte[] secretsFileContent = readableContents.get(0);
+ if (secretsFileContent == null) {
+ return;
+ }
+ Properties userAndPass = new Properties();
+ userAndPass.load(new ByteArrayInputStream(secretsFileContent));
+ config.setUser(userAndPass.getProperty("user"));
+ config.setPassword(userAndPass.getProperty("password"));
+
+ if (elasticSearch7Client == null) {
+ //In the startup process, we just need to change the username/password
+ } else {
+ elasticSearch7Client.connect();
+ }
+ }, config.getSecretsManagementFile());
+ /**
+ * By leveraging the sync update check feature when startup.
+ */
+ monitor.start();
+ }
+
elasticSearch7Client = new ElasticSearch7Client(
config.getClusterNodes(), config.getProtocol(), config.getTrustStorePath(), config
.getTrustStorePass(), config.getUser(), config.getPassword(),
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/client/ElasticSearch7Client.java b/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/client/ElasticSearch7Client.java
index 87b62d8..e561c82 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/client/ElasticSearch7Client.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/client/ElasticSearch7Client.java
@@ -27,6 +27,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient;
@@ -62,13 +63,12 @@ import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.index.reindex.DeleteByQueryRequest;
import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+/**
+ *
+ */
+@Slf4j
public class ElasticSearch7Client extends ElasticSearchClient {
-
- private static final Logger logger = LoggerFactory.getLogger(ElasticSearch7Client.class);
-
public ElasticSearch7Client(final String clusterNodes,
final String protocol,
final String trustStorePath,
@@ -84,6 +84,13 @@ public class ElasticSearch7Client extends ElasticSearchClient {
@Override
public void connect() throws IOException, KeyStoreException, NoSuchAlgorithmException, KeyManagementException, CertificateException {
+ if (client != null) {
+ try {
+ client.close();
+ } catch (Throwable t) {
+ log.error("ElasticSearch7 client reconnection fails based on new config", t);
+ }
+ }
List<HttpHost> hosts = parseClusterNodes(protocol, clusterNodes);
client = createClient(hosts);
client.ping(RequestOptions.DEFAULT);
@@ -94,7 +101,7 @@ public class ElasticSearch7Client extends ElasticSearchClient {
CreateIndexRequest request = new CreateIndexRequest(indexName);
CreateIndexResponse response = client.indices().create(request, RequestOptions.DEFAULT);
- logger.debug("create {} index finished, isAcknowledged: {}", indexName, response.isAcknowledged());
+ log.debug("create {} index finished, isAcknowledged: {}", indexName, response.isAcknowledged());
return response.isAcknowledged();
}
@@ -105,7 +112,7 @@ public class ElasticSearch7Client extends ElasticSearchClient {
request.settings(settings);
request.mapping(mapping);
CreateIndexResponse response = client.indices().create(request, RequestOptions.DEFAULT);
- logger.debug("create {} index finished, isAcknowledged: {}", indexName, response.isAcknowledged());
+ log.debug("create {} index finished, isAcknowledged: {}", indexName, response.isAcknowledged());
return response.isAcknowledged();
}
@@ -127,7 +134,7 @@ public class ElasticSearch7Client extends ElasticSearchClient {
}
DeleteIndexRequest request = new DeleteIndexRequest(indexName);
AcknowledgedResponse response = client.indices().delete(request, RequestOptions.DEFAULT);
- logger.debug("delete {} index finished, isAcknowledged: {}", indexName, response.isAcknowledged());
+ log.debug("delete {} index finished, isAcknowledged: {}", indexName, response.isAcknowledged());
return response.isAcknowledged();
}
@@ -234,7 +241,7 @@ public class ElasticSearch7Client extends ElasticSearchClient {
deleteByQueryRequest.setAbortOnVersionConflict(false);
deleteByQueryRequest.setQuery(QueryBuilders.rangeQuery(timeBucketColumnName).lte(endTimeBucket));
BulkByScrollResponse bulkByScrollResponse = client.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT);
- logger.debug(
+ log.debug(
"delete indexName: {}, by query request: {}, response: {}", indexName, deleteByQueryRequest,
bulkByScrollResponse
);
@@ -248,9 +255,9 @@ public class ElasticSearch7Client extends ElasticSearchClient {
try {
int size = request.requests().size();
BulkResponse responses = client.bulk(request, RequestOptions.DEFAULT);
- logger.info("Synchronous bulk took time: {} millis, size: {}", responses.getTook().getMillis(), size);
+ log.info("Synchronous bulk took time: {} millis, size: {}", responses.getTook().getMillis(), size);
} catch (IOException e) {
- logger.error(e.getMessage(), e);
+ log.error(e.getMessage(), e);
}
}