You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by na...@apache.org on 2021/06/11 15:35:35 UTC
[ignite] branch master updated: IGNITE-14658 Add client connector
SSL metrics. (#9132)
This is an automated email from the ASF dual-hosted git repository.
namelchev pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 3f65264 IGNITE-14658 Add client connector SSL metrics. (#9132)
3f65264 is described below
commit 3f6526426b8e9357fd589f24cae995b22f99246d
Author: Mikhail Petrov <32...@users.noreply.github.com>
AuthorDate: Fri Jun 11 18:35:13 2021 +0300
IGNITE-14658 Add client connector SSL metrics. (#9132)
---
.../ignite/common/NodeSslConnectionMetricTest.java | 448 +++++++++++++++++++++
.../client/suite/IgniteClientTestSuite.java | 4 +-
.../GridClientConnectionManagerAdapter.java | 2 +-
.../client/router/impl/GridTcpRouterImpl.java | 2 +-
.../GridNioClientConnectionMultiplexer.java | 2 +-
.../processors/odbc/ClientListenerProcessor.java | 6 +-
.../rest/protocols/tcp/GridTcpRestProtocol.java | 14 +-
.../ignite/internal/util/nio/GridNioServer.java | 14 +
.../internal/util/nio/ssl/GridNioSslFilter.java | 56 ++-
.../tcp/internal/GridNioServerWrapper.java | 9 +-
.../ignite/spi/discovery/tcp/ServerImpl.java | 5 +-
.../ignite/spi/discovery/tcp/TcpDiscoverySpi.java | 2 +
.../tcp/internal/TcpDiscoveryStatistics.java | 15 +
.../internal/util/nio/GridNioSslSelfTest.java | 2 +-
14 files changed, 568 insertions(+), 13 deletions(-)
diff --git a/modules/clients/src/test/java/org/apache/ignite/common/NodeSslConnectionMetricTest.java b/modules/clients/src/test/java/org/apache/ignite/common/NodeSslConnectionMetricTest.java
new file mode 100644
index 0000000..99b1904
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/common/NodeSslConnectionMetricTest.java
@@ -0,0 +1,448 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.common;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Collections;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.client.ClientConnectionException;
+import org.apache.ignite.client.Config;
+import org.apache.ignite.client.IgniteClient;
+import org.apache.ignite.client.SslMode;
+import org.apache.ignite.configuration.ClientConfiguration;
+import org.apache.ignite.configuration.ClientConnectorConfiguration;
+import org.apache.ignite.configuration.ConnectorConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.client.GridClient;
+import org.apache.ignite.internal.client.GridClientConfiguration;
+import org.apache.ignite.internal.client.GridClientFactory;
+import org.apache.ignite.internal.processors.metric.MetricRegistry;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.metric.BooleanMetric;
+import org.apache.ignite.spi.metric.HistogramMetric;
+import org.apache.ignite.spi.metric.IntMetric;
+import org.apache.ignite.spi.metric.LongMetric;
+import org.apache.ignite.ssl.SslContextFactory;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+import static java.sql.DriverManager.getConnection;
+import static org.apache.ignite.Ignition.startClient;
+import static org.apache.ignite.internal.client.GridClientFactory.start;
+import static org.apache.ignite.internal.managers.discovery.GridDiscoveryManager.DISCO_METRICS;
+import static org.apache.ignite.internal.processors.odbc.ClientListenerProcessor.CLIENT_CONNECTOR_METRIC_REGISTRY_NAME;
+import static org.apache.ignite.internal.processors.rest.protocols.tcp.GridTcpRestProtocol.REST_CONNECTOR_METRIC_REGISTRY_NAME;
+import static org.apache.ignite.internal.util.nio.GridNioServer.RECEIVED_BYTES_METRIC_NAME;
+import static org.apache.ignite.internal.util.nio.GridNioServer.SENT_BYTES_METRIC_NAME;
+import static org.apache.ignite.internal.util.nio.GridNioServer.SESSIONS_CNT_METRIC_NAME;
+import static org.apache.ignite.internal.util.nio.GridNioServer.SSL_ENABLED_METRIC_NAME;
+import static org.apache.ignite.internal.util.nio.ssl.GridNioSslFilter.SSL_HANDSHAKE_DURATION_HISTOGRAM_METRIC_NAME;
+import static org.apache.ignite.internal.util.nio.ssl.GridNioSslFilter.SSL_REJECTED_SESSIONS_CNT_METRIC_NAME;
+import static org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.COMMUNICATION_METRICS_GROUP_NAME;
+import static org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause;
+import static org.apache.ignite.testframework.GridTestUtils.keyStorePassword;
+import static org.apache.ignite.testframework.GridTestUtils.keyStorePath;
+import static org.apache.ignite.testframework.GridTestUtils.sslTrustedFactory;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+
+/** Checks SSL metrics for various node connection approaches. */
+public class NodeSslConnectionMetricTest extends GridCommonAbstractTest {
+ /** Cipher suite supported by cluster nodes. */
+ private static final String CIPHER_SUITE = "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256";
+
+ /** Cipher suite not supported by cluster nodes. */
+ private static final String UNSUPPORTED_CIPHER_SUITE = "TLS_RSA_WITH_AES_128_GCM_SHA256";
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ GridClientFactory.stopAll(false);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids(true);
+ }
+
+ /** Checks the status of the SSL metric if SSL is not configured on the node. */
+ @Test
+ public void testSslDisabled() throws Exception {
+ IgniteEx srv = startGrid();
+
+ MetricRegistry discoReg = mreg(srv, DISCO_METRICS);
+
+ assertFalse(discoReg.<BooleanMetric>findMetric("SslEnabled").value());
+ assertEquals(0, discoReg.<IntMetric>findMetric("RejectedSslConnectionsCount").value());
+
+ MetricRegistry commReg = mreg(srv, COMMUNICATION_METRICS_GROUP_NAME);
+
+ assertFalse(commReg.<BooleanMetric>findMetric(SSL_ENABLED_METRIC_NAME).value());
+ assertNull(commReg.<IntMetric>findMetric(SSL_REJECTED_SESSIONS_CNT_METRIC_NAME));
+ assertNull(commReg.<HistogramMetric>findMetric(SSL_HANDSHAKE_DURATION_HISTOGRAM_METRIC_NAME));
+ assertEquals(0, commReg.<IntMetric>findMetric(SESSIONS_CNT_METRIC_NAME).value());
+
+ MetricRegistry cliConnReg = mreg(srv, CLIENT_CONNECTOR_METRIC_REGISTRY_NAME);
+
+ assertFalse(cliConnReg.<BooleanMetric>findMetric(SSL_ENABLED_METRIC_NAME).value());
+ assertNull(cliConnReg.<IntMetric>findMetric(SSL_REJECTED_SESSIONS_CNT_METRIC_NAME));
+ assertNull(cliConnReg.<HistogramMetric>findMetric(SSL_HANDSHAKE_DURATION_HISTOGRAM_METRIC_NAME));
+ assertEquals(0, cliConnReg.<IntMetric>findMetric(SESSIONS_CNT_METRIC_NAME).value());
+
+ MetricRegistry restConnReg = mreg(srv, REST_CONNECTOR_METRIC_REGISTRY_NAME);
+
+ assertNull(restConnReg.<BooleanMetric>findMetric(SSL_ENABLED_METRIC_NAME));
+ assertNull(restConnReg.<IntMetric>findMetric(SSL_REJECTED_SESSIONS_CNT_METRIC_NAME));
+ assertNull(restConnReg.<HistogramMetric>findMetric(SSL_HANDSHAKE_DURATION_HISTOGRAM_METRIC_NAME));
+ assertNull(restConnReg.<IntMetric>findMetric(SESSIONS_CNT_METRIC_NAME));
+
+ stopAllGrids();
+
+ srv = startGrid(getConfiguration().setConnectorConfiguration(new ConnectorConfiguration()));
+
+ restConnReg = mreg(srv, REST_CONNECTOR_METRIC_REGISTRY_NAME);
+
+ assertFalse(restConnReg.<BooleanMetric>findMetric(SSL_ENABLED_METRIC_NAME).value());
+ assertEquals(0, restConnReg.<IntMetric>findMetric(SESSIONS_CNT_METRIC_NAME).value());
+ }
+
+ /** Tests SSL metrics produced by JDBC connection. */
+ @Test
+ public void testJdbc() throws Exception {
+ MetricRegistry reg = mreg(startClusterNode(0), CLIENT_CONNECTOR_METRIC_REGISTRY_NAME);
+
+ assertEquals(0, reg.<LongMetric>findMetric(SENT_BYTES_METRIC_NAME).value());
+ assertEquals(0, reg.<LongMetric>findMetric(RECEIVED_BYTES_METRIC_NAME).value());
+
+ try (Connection ignored = getConnection(jdbcConfiguration("thinClient", "trusttwo", CIPHER_SUITE, "TLSv1.2"))) {
+ checkSslCommunicationMetrics(reg, 1, 1, 0);
+ }
+
+ assertTrue(reg.<LongMetric>findMetric(SENT_BYTES_METRIC_NAME).value() > 0);
+ assertTrue(reg.<LongMetric>findMetric(RECEIVED_BYTES_METRIC_NAME).value() > 0);
+
+ checkSslCommunicationMetrics(reg, 1, 0, 0);
+
+ // Tests untrusted certificate.
+ assertThrowsWithCause(() ->
+ getConnection(jdbcConfiguration("client", "trusttwo", CIPHER_SUITE, "TLSv1.2")),
+ SQLException.class);
+
+ checkSslCommunicationMetrics(reg, 2, 0, 1);
+
+ // Tests unsupported cipher suite.
+ assertThrowsWithCause(() ->
+ getConnection(jdbcConfiguration("thinClient", "trusttwo", UNSUPPORTED_CIPHER_SUITE, "TLSv1.2")),
+ SQLException.class);
+
+ checkSslCommunicationMetrics(reg, 3, 0, 2);
+
+ assertThrowsWithCause(() ->
+ getConnection(jdbcConfiguration("thinClient", "trusttwo", null, "TLSv1.1")),
+ SQLException.class);
+
+ checkSslCommunicationMetrics(reg, 4, 0, 3);
+ }
+
+ /** Tests SSL metrics produced by REST TCP client connection. */
+ @Test
+ public void testRestClientConnector() throws Exception {
+ MetricRegistry reg = mreg(startClusterNode(0), REST_CONNECTOR_METRIC_REGISTRY_NAME);
+
+ assertEquals(0, reg.<LongMetric>findMetric(SENT_BYTES_METRIC_NAME).value());
+ assertEquals(0, reg.<LongMetric>findMetric(RECEIVED_BYTES_METRIC_NAME).value());
+
+ try (
+ GridClient ignored = start(gridClientConfiguration("connectorClient", "trustthree", CIPHER_SUITE, "TLSv1.2"))
+ ) {
+ checkSslCommunicationMetrics(reg, 1, 1, 0);
+ }
+
+ assertTrue(reg.<LongMetric>findMetric(SENT_BYTES_METRIC_NAME).value() > 0);
+ assertTrue(reg.<LongMetric>findMetric(RECEIVED_BYTES_METRIC_NAME).value() > 0);
+
+ checkSslCommunicationMetrics(reg, 1, 0, 0);
+
+ // Tests untrusted certificate.
+ try (GridClient ignored = start(gridClientConfiguration("client", "trustthree", CIPHER_SUITE, "TLSv1.2"))) {
+ // GridClient makes 2 additional silent connection attempts if an SSL error occurs.
+ }
+
+ checkSslCommunicationMetrics(reg, 4, 0, 3);
+
+ // Tests unsupported cipher suite.
+ try (
+ GridClient ignored = start(gridClientConfiguration("connectorClient", "trustthree",
+ UNSUPPORTED_CIPHER_SUITE, "TLSv1.2"))
+ ) {
+ // GridClient makes 2 additional silent connection attempts if an SSL error occurs.
+ }
+
+ checkSslCommunicationMetrics(reg, 7, 0, 6);
+
+ // Tests mismatched protocol versions.
+ try (GridClient ignored = start(gridClientConfiguration("connectorClient", "trustthree", null, "TLSv1.1"))) {
+ // GridClient makes 2 additional silent connection attempts if an SSL error occurs.
+ }
+
+ checkSslCommunicationMetrics(reg, 10, 0, 9);
+ }
+
+ /** Tests SSL discovery metrics produced by node connection. */
+ @Test
+ public void testDiscovery() throws Exception {
+ MetricRegistry reg = mreg(startClusterNode(0), DISCO_METRICS);
+
+ startGrid(nodeConfiguration(1, true, "client", "trustone", CIPHER_SUITE, "TLSv1.2"));
+
+ assertTrue(reg.<BooleanMetric>findMetric("SslEnabled").value());
+ assertEquals(0, reg.<IntMetric>findMetric("RejectedSslConnectionsCount").value());
+
+ // Tests untrusted certificate.
+ checkNodeJoinFails(2, true, "thinClient", "trusttwo", CIPHER_SUITE, "TLSv1.2");
+ checkNodeJoinFails(2, false, "thinClient", "trusttwo", CIPHER_SUITE, "TLSv1.2");
+ // Tests untrusted cipher suites.
+ checkNodeJoinFails(2, true, "client", "trustone", UNSUPPORTED_CIPHER_SUITE, "TLSv1.2");
+ checkNodeJoinFails(2, false, "node01", "trustone", UNSUPPORTED_CIPHER_SUITE, "TLSv1.2");
+
+ // Tests mismatched protocol versions.
+ checkNodeJoinFails(2, true, "client", "trustone", null, "TLSv1.1");
+ checkNodeJoinFails(2, false, "node01", "trustone", null, "TLSv1.1");
+
+ // In case of an SSL error, the client and server nodes make 2 additional connection attempts.
+ assertTrue(waitForCondition(() ->
+ 18 == reg.<IntMetric>findMetric("RejectedSslConnectionsCount").value(),
+ getTestTimeout()));
+ }
+
+ /** Tests SSL communication metrics produced by node connection. */
+ @Test
+ public void testCommunication() throws Exception {
+ MetricRegistry reg = mreg(startClusterNode(0), COMMUNICATION_METRICS_GROUP_NAME);
+
+ assertEquals(0, reg.<LongMetric>findMetric(SENT_BYTES_METRIC_NAME).value());
+ assertEquals(0, reg.<LongMetric>findMetric(RECEIVED_BYTES_METRIC_NAME).value());
+
+ checkSslCommunicationMetrics(reg, 0, 0, 0);
+
+ try (
+ IgniteEx cliNode = startGrid(nodeConfiguration(1, true, "client", "trustone", CIPHER_SUITE, "TLSv1.2"));
+ IgniteEx srvNode = startGrid(nodeConfiguration(2, false, "node01", "trustone", CIPHER_SUITE, "TLSv1.2"))
+ ) {
+ checkSslCommunicationMetrics(reg, 2, 2, 0);
+
+ MetricRegistry cliNodeReg = mreg(cliNode, COMMUNICATION_METRICS_GROUP_NAME);
+
+ checkSslCommunicationMetrics(cliNodeReg, 0, 1, 0);
+
+ assertTrue(cliNodeReg.<LongMetric>findMetric(SENT_BYTES_METRIC_NAME).value() > 0);
+ assertTrue(cliNodeReg.<LongMetric>findMetric(RECEIVED_BYTES_METRIC_NAME).value() > 0);
+
+ MetricRegistry srvNodeReg = mreg(srvNode, COMMUNICATION_METRICS_GROUP_NAME);
+
+ checkSslCommunicationMetrics(srvNodeReg, 0, 1, 0);
+
+ assertTrue(srvNodeReg.<LongMetric>findMetric(SENT_BYTES_METRIC_NAME).value() > 0);
+ assertTrue(srvNodeReg.<LongMetric>findMetric(RECEIVED_BYTES_METRIC_NAME).value() > 0);
+ }
+
+ assertTrue(reg.<LongMetric>findMetric(SENT_BYTES_METRIC_NAME).value() > 0);
+ assertTrue(reg.<LongMetric>findMetric(RECEIVED_BYTES_METRIC_NAME).value() > 0);
+
+ checkSslCommunicationMetrics(reg, 2, 0, 0);
+ }
+
+ /** Tests SSL metrics produced by thin client connection. */
+ @Test
+ public void testClientConnector() throws Exception {
+ MetricRegistry reg = mreg(startClusterNode(0), CLIENT_CONNECTOR_METRIC_REGISTRY_NAME);
+
+ assertEquals(0, reg.<LongMetric>findMetric(SENT_BYTES_METRIC_NAME).value());
+ assertEquals(0, reg.<LongMetric>findMetric(RECEIVED_BYTES_METRIC_NAME).value());
+
+ try (IgniteClient ignored = startClient(clientConfiguration("thinClient", "trusttwo", CIPHER_SUITE, "TLSv1.2"))) {
+ checkSslCommunicationMetrics(reg, 1, 1, 0);
+ }
+
+ assertTrue(reg.<LongMetric>findMetric(SENT_BYTES_METRIC_NAME).value() > 0);
+ assertTrue(reg.<LongMetric>findMetric(RECEIVED_BYTES_METRIC_NAME).value() > 0);
+
+ checkSslCommunicationMetrics(reg, 1, 0, 0);
+
+ // Tests untrusted certificate.
+ assertThrowsWithCause(() ->
+ startClient(clientConfiguration("client", "trustboth", CIPHER_SUITE, "TLSv1.2")),
+ ClientConnectionException.class);
+
+ checkSslCommunicationMetrics(reg, 2, 0, 1);
+
+ // Tests unsupported cipher suites.
+ assertThrowsWithCause(() ->
+ startClient(clientConfiguration("thinClient", "trusttwo", UNSUPPORTED_CIPHER_SUITE, "TLSv1.2")),
+ ClientConnectionException.class
+ );
+
+ checkSslCommunicationMetrics(reg, 3, 0, 2);
+
+ // Tests mismatched protocol versions.
+ assertThrowsWithCause(() ->
+ startClient(clientConfiguration("thinClient", "trusttwo", null, "TLSv1.1")),
+ ClientConnectionException.class
+ );
+
+ checkSslCommunicationMetrics(reg, 4, 0, 3);
+ }
+
+ /** Starts node that imitates a cluster server node to which connections will be performed. */
+ private IgniteEx startClusterNode(int idx) throws Exception {
+ IgniteConfiguration cfg = getConfiguration(getTestIgniteInstanceName(idx));
+
+ cfg.setSslContextFactory(sslContextFactory("server", "trustone", CIPHER_SUITE, "TLSv1.2"));
+
+ cfg.setClientConnectorConfiguration(new ClientConnectorConfiguration()
+ .setSslEnabled(true)
+ .setSslClientAuth(true)
+ .setUseIgniteSslContextFactory(false)
+ .setSslContextFactory(sslContextFactory("thinServer", "trusttwo", CIPHER_SUITE, "TLSv1.2")));
+
+ cfg.setConnectorConfiguration(new ConnectorConfiguration()
+ .setSslClientAuth(true)
+ .setSslEnabled(true)
+ .setSslFactory(sslContextFactory("connectorServer", "trustthree", CIPHER_SUITE, "TLSv1.2")));
+
+ return startGrid(cfg);
+ }
+
+ /** @return JDBC connection configuration with specified SSL options. */
+ private String jdbcConfiguration(String keyStore, String trustStore, String cipherSuite, String protocol) {
+ String res = "jdbc:ignite:thin://" + Config.SERVER + "?sslMode=require" +
+ "&sslClientCertificateKeyStoreUrl=" + keyStorePath(keyStore) +
+ "&sslClientCertificateKeyStorePassword=" + keyStorePassword() +
+ "&sslTrustCertificateKeyStoreUrl=" + keyStorePath(trustStore) +
+ "&sslTrustCertificateKeyStorePassword=" + keyStorePassword() +
+ "&sslProtocol=" + protocol;
+
+ if (cipherSuite != null)
+ res += "&sslCipherSuites=" + cipherSuite;
+
+ return res;
+ }
+
+ /** @return Node connection configuration with specified SSL options. */
+ private IgniteConfiguration nodeConfiguration(
+ int idx,
+ boolean client,
+ String keyStore,
+ String trustStore,
+ String cipherSuite,
+ String protocol
+ ) throws Exception {
+ return getConfiguration(getTestIgniteInstanceName(idx))
+ .setSslContextFactory(sslContextFactory(keyStore, trustStore, cipherSuite, protocol))
+ .setClientMode(client);
+ }
+
+ /** @return Grid client connection configuration with specified SSL options. */
+ private GridClientConfiguration gridClientConfiguration(
+ String keyStore,
+ String trustStore,
+ String cipherSuite,
+ String protocol
+ ) {
+ SslContextFactory sslCtxFactory = sslContextFactory(keyStore, trustStore, cipherSuite, protocol);
+
+ return new GridClientConfiguration()
+ .setServers(Collections.singleton("127.0.0.1:11211"))
+ .setSslContextFactory(sslCtxFactory::create);
+ }
+
+ /** @return Thin client connection configuration with specified SSL options. */
+ private ClientConfiguration clientConfiguration(
+ String keyStore,
+ String trustStore,
+ String cipherSuite,
+ String protocol
+ ) {
+ return new ClientConfiguration()
+ .setAddresses("127.0.0.1:10800")
+ .setSslMode(SslMode.REQUIRED)
+ .setSslContextFactory(sslContextFactory(keyStore, trustStore, cipherSuite, protocol));
+ }
+
+ /** Checks that the node join failed if the connection was performed with the specified SSL options. */
+ @SuppressWarnings("ThrowableNotThrown")
+ private void checkNodeJoinFails(
+ int idx,
+ boolean client,
+ String keyStore,
+ String trustStore,
+ String cipherSuite,
+ String protocol
+ ) throws Exception {
+ IgniteConfiguration cfg = nodeConfiguration(idx, client, keyStore, trustStore, cipherSuite, protocol);
+
+ if (client)
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setJoinTimeout(1); // To prevent client multiple join attempts.
+
+ GridTestUtils.assertThrowsWithCause(() -> startGrid(cfg), IgniteCheckedException.class);
+ }
+
+ /** Obtains the metric registry with the specified name from Ignite instance. */
+ private MetricRegistry mreg(IgniteEx ignite, String name) {
+ return ignite.context().metric().registry(name);
+ }
+
+ /** Checks SSL communication metrics. */
+ private void checkSslCommunicationMetrics(
+ MetricRegistry mreg,
+ long handshakeCnt,
+ int sesCnt,
+ int rejectedSesCnt
+ ) throws Exception {
+ assertEquals(true, mreg.<BooleanMetric>findMetric(SSL_ENABLED_METRIC_NAME).value());
+ assertTrue(waitForCondition(() ->
+ sesCnt == mreg.<IntMetric>findMetric(SESSIONS_CNT_METRIC_NAME).value(),
+ getTestTimeout()));
+ assertTrue(waitForCondition(() ->
+ handshakeCnt == Arrays.stream(
+ mreg.<HistogramMetric>findMetric(SSL_HANDSHAKE_DURATION_HISTOGRAM_METRIC_NAME).value()
+ ).sum(),
+ getTestTimeout()));
+ assertTrue(waitForCondition(() ->
+ rejectedSesCnt == mreg.<IntMetric>findMetric(SSL_REJECTED_SESSIONS_CNT_METRIC_NAME).value(),
+ getTestTimeout()));
+ }
+
+ /** Creates {@link SslContextFactory} with specified options. */
+ private SslContextFactory sslContextFactory(String keyStore, String trustStore, String cipherSuite, String protocol) {
+ SslContextFactory res = (SslContextFactory)sslTrustedFactory(keyStore, trustStore);
+
+ if (cipherSuite != null)
+ res.setCipherSuites(cipherSuite);
+
+ res.setProtocols(protocol);
+
+ return res;
+ }
+}
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java
index 222cba3..0a50e40 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.client.suite;
import org.apache.ignite.common.ClientSideCacheCreationDestructionWileTopologyChangeTest;
import org.apache.ignite.common.ClientSizeCacheCreationDestructionTest;
+import org.apache.ignite.common.NodeSslConnectionMetricTest;
import org.apache.ignite.internal.IgniteClientFailuresTest;
import org.apache.ignite.internal.TaskEventSubjectIdSelfTest;
import org.apache.ignite.internal.client.ClientDefaultCacheSelfTest;
@@ -182,7 +183,8 @@ import org.junit.runners.Suite;
IgniteClientFailuresTest.class,
ClientSizeCacheCreationDestructionTest.class,
- ClientSideCacheCreationDestructionWileTopologyChangeTest.class
+ ClientSideCacheCreationDestructionWileTopologyChangeTest.class,
+ NodeSslConnectionMetricTest.class
})
public class IgniteClientTestSuite {
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
index 63f19c4..76bab83 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
@@ -189,7 +189,7 @@ public abstract class GridClientConnectionManagerAdapter implements GridClientCo
GridNioFilter codecFilter = new GridNioCodecFilter(new GridTcpRestParser(routerClient), gridLog, false);
if (sslCtx != null) {
- GridNioSslFilter sslFilter = new GridNioSslFilter(sslCtx, true, ByteOrder.nativeOrder(), gridLog);
+ GridNioSslFilter sslFilter = new GridNioSslFilter(sslCtx, true, ByteOrder.nativeOrder(), gridLog, null);
sslFilter.directMode(false);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java
index d6daf99..4e06092 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java
@@ -240,7 +240,7 @@ public class GridTcpRouterImpl implements GridTcpRouter, GridTcpRouterMBean, Lif
GridNioFilter[] filters;
if (sslCtx != null) {
- GridNioSslFilter sslFilter = new GridNioSslFilter(sslCtx, false, ByteOrder.nativeOrder(), log);
+ GridNioSslFilter sslFilter = new GridNioSslFilter(sslCtx, false, ByteOrder.nativeOrder(), log, null);
sslFilter.wantClientAuth(wantClientAuth);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/io/gridnioserver/GridNioClientConnectionMultiplexer.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/io/gridnioserver/GridNioClientConnectionMultiplexer.java
index 74a7025..4a041f9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/io/gridnioserver/GridNioClientConnectionMultiplexer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/io/gridnioserver/GridNioClientConnectionMultiplexer.java
@@ -75,7 +75,7 @@ public class GridNioClientConnectionMultiplexer implements ClientConnectionMulti
sslCtx = ClientSslUtils.getSslContext(cfg);
if (sslCtx != null) {
- GridNioSslFilter sslFilter = new GridNioSslFilter(sslCtx, true, ByteOrder.nativeOrder(), gridLog);
+ GridNioSslFilter sslFilter = new GridNioSslFilter(sslCtx, true, ByteOrder.nativeOrder(), gridLog, null);
sslFilter.directMode(false);
filters = new GridNioFilter[] {codecFilter, sslFilter};
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerProcessor.java
index e25512a..cce21d2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerProcessor.java
@@ -72,6 +72,9 @@ public class ClientListenerProcessor extends GridProcessorAdapter {
/** */
public static final String CLI_CONN_VIEW_DESC = "Client connections";
+ /** The name of the metric registry associated with the thin client connector. */
+ public static final String CLIENT_CONNECTOR_METRIC_REGISTRY_NAME = metricName("connector", "client", "thin", "tcp");
+
/** Default client connector configuration. */
public static final ClientConnectorConfiguration DFLT_CLI_CFG = new ClientConnectorConfigurationEx();
@@ -177,6 +180,7 @@ public class ClientListenerProcessor extends GridProcessorAdapter {
.filters(filters)
.directMode(true)
.idleTimeout(idleTimeout > 0 ? idleTimeout : Long.MAX_VALUE)
+ .metricRegistry(ctx.metric().registry(CLIENT_CONNECTOR_METRIC_REGISTRY_NAME))
.build();
ctx.ports().registerPort(port, IgnitePortProtocol.TCP, getClass());
@@ -334,7 +338,7 @@ public class ClientListenerProcessor extends GridProcessorAdapter {
"(SSL is enabled but factory is null). Check the ClientConnectorConfiguration");
GridNioSslFilter sslFilter = new GridNioSslFilter(sslCtxFactory.create(),
- true, ByteOrder.nativeOrder(), log);
+ true, ByteOrder.nativeOrder(), log, ctx.metric().registry(CLIENT_CONNECTOR_METRIC_REGISTRY_NAME));
sslFilter.directMode(true);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java
index afb5525..081e461 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java
@@ -53,6 +53,8 @@ import org.apache.ignite.plugin.PluginProvider;
import org.apache.ignite.spi.IgnitePortProtocol;
import org.jetbrains.annotations.Nullable;
+import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.metricName;
+
/**
* TCP binary protocol implementation.
*/
@@ -63,6 +65,9 @@ public class GridTcpRestProtocol extends GridRestProtocolAdapter {
/** NIO server listener. */
private GridTcpRestNioListener lsnr;
+ /** The name of the metric registry associated with the REST TCP connector. */
+ public static final String REST_CONNECTOR_METRIC_REGISTRY_NAME = metricName("connector", "client", "rest", "tcp");
+
/** @param ctx Context. */
public GridTcpRestProtocol(GridKernalContext ctx) {
super(ctx);
@@ -210,8 +215,12 @@ public class GridTcpRestProtocol extends GridRestProtocolAdapter {
GridNioFilter[] filters;
if (sslCtx != null) {
- GridNioSslFilter sslFilter = new GridNioSslFilter(sslCtx,
- cfg.isDirectBuffer(), ByteOrder.nativeOrder(), log);
+ GridNioSslFilter sslFilter = new GridNioSslFilter(
+ sslCtx,
+ cfg.isDirectBuffer(),
+ ByteOrder.nativeOrder(),
+ log,
+ ctx.metric().registry(REST_CONNECTOR_METRIC_REGISTRY_NAME));
sslFilter.directMode(false);
@@ -245,6 +254,7 @@ public class GridTcpRestProtocol extends GridRestProtocolAdapter {
.sendQueueLimit(cfg.getSendQueueLimit())
.filters(filters)
.directMode(false)
+ .metricRegistry(ctx.metric().registry(REST_CONNECTOR_METRIC_REGISTRY_NAME))
.build();
srv.idleTimeout(cfg.getIdleTimeout());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index 32e0a67..9da0493 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -163,6 +163,12 @@ public class GridNioServer<T> {
/** */
public static final String SENT_BYTES_METRIC_DESC = "Total number of bytes sent by current node";
+ /** The name of the metric that indicates whether SSL is enabled for the connector. */
+ public static final String SSL_ENABLED_METRIC_NAME = "SslEnabled";
+
+ /** The name of the metric that provides the active TCP sessions count. */
+ public static final String SESSIONS_CNT_METRIC_NAME = "ActiveSessionsCount";
+
/** Defines how many times selector should do {@code selectNow()} before doing {@code select(long)}. */
private long selectorSpins;
@@ -451,6 +457,14 @@ public class GridNioServer<T> {
OUTBOUND_MESSAGES_QUEUE_SIZE_METRIC_NAME,
OUTBOUND_MESSAGES_QUEUE_SIZE_METRIC_DESC
);
+
+ if (mreg != null) {
+ mreg.register(SESSIONS_CNT_METRIC_NAME, sessions::size, "Active TCP sessions count.");
+
+ boolean sslEnabled = Arrays.stream(filters).anyMatch(filter -> filter instanceof GridNioSslFilter);
+
+ mreg.register(SSL_ENABLED_METRIC_NAME, () -> sslEnabled, "Whether SSL is enabled");
+ }
}
/**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslFilter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslFilter.java
index 7167c40..dc4ca58 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslFilter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslFilter.java
@@ -26,6 +26,9 @@ import javax.net.ssl.SSLException;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.processors.metric.MetricRegistry;
+import org.apache.ignite.internal.processors.metric.impl.HistogramMetricImpl;
+import org.apache.ignite.internal.processors.metric.impl.IntMetricImpl;
import org.apache.ignite.internal.util.nio.GridNioException;
import org.apache.ignite.internal.util.nio.GridNioFilterAdapter;
import org.apache.ignite.internal.util.nio.GridNioFinishedFuture;
@@ -35,6 +38,7 @@ import org.apache.ignite.internal.util.nio.GridNioSession;
import org.apache.ignite.internal.util.nio.GridNioSessionMetaKey;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteInClosure;
+import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.SSL_META;
@@ -45,6 +49,12 @@ public class GridNioSslFilter extends GridNioFilterAdapter {
/** SSL handshake future metadata key. */
public static final int HANDSHAKE_FUT_META_KEY = GridNioSessionMetaKey.nextUniqueKey();
+ /** The name of the metric that provides histogram of SSL handshake duration. */
+ public static final String SSL_HANDSHAKE_DURATION_HISTOGRAM_METRIC_NAME = "SslHandshakeDurationHistogram";
+
+ /** The name of the metric that provides sessions count that were rejected due to SSL errors. */
+ public static final String SSL_REJECTED_SESSIONS_CNT_METRIC_NAME = "RejectedSslSessionsCount";
+
/** Logger to use. */
private IgniteLogger log;
@@ -72,6 +82,12 @@ public class GridNioSslFilter extends GridNioFilterAdapter {
/** Whether direct mode is used. */
private boolean directMode;
+ /** Metric that indicates sessions count that were rejected due to SSL errors. */
+ @Nullable private final IntMetricImpl rejectedSesCnt;
+
+ /** Histogram that provides distribution of SSL handshake duration. */
+ @Nullable private final HistogramMetricImpl handshakeDuration;
+
/**
* Creates SSL filter.
*
@@ -79,14 +95,32 @@ public class GridNioSslFilter extends GridNioFilterAdapter {
* @param directBuf Direct buffer flag.
* @param order Byte order.
* @param log Logger to use.
+ * @param mreg Optional metric registry.
*/
- public GridNioSslFilter(SSLContext sslCtx, boolean directBuf, ByteOrder order, IgniteLogger log) {
+ public GridNioSslFilter(
+ SSLContext sslCtx,
+ boolean directBuf,
+ ByteOrder order,
+ IgniteLogger log,
+ @Nullable MetricRegistry mreg
+ ) {
super("SSL filter");
this.log = log;
this.sslCtx = sslCtx;
this.directBuf = directBuf;
this.order = order;
+
+ handshakeDuration = mreg == null ? null : mreg.histogram(
+ SSL_HANDSHAKE_DURATION_HISTOGRAM_METRIC_NAME,
+ new long[] {250, 500, 1000},
+ "SSL handshake duration in milliseconds."
+ );
+
+ rejectedSesCnt = mreg == null ? null : mreg.intMetric(
+ SSL_REJECTED_SESSIONS_CNT_METRIC_NAME,
+ "TCP sessions count that were rejected due to SSL errors."
+ );
}
/**
@@ -201,6 +235,20 @@ public class GridNioSslFilter extends GridNioFilterAdapter {
sslMeta.handler(hnd);
+ if (handshakeDuration != null) {
+ GridNioFutureImpl<?> fut = ses.meta(HANDSHAKE_FUT_META_KEY);
+
+ if (fut == null) {
+ fut = new GridNioFutureImpl<>(null);
+
+ ses.addMeta(HANDSHAKE_FUT_META_KEY, fut);
+ }
+
+ long startTime = System.nanoTime();
+
+ fut.listen(f -> handshakeDuration.value(U.nanosToMillis(System.nanoTime() - startTime)));
+ }
+
hnd.handshake();
ByteBuffer alreadyDecoded = sslMeta.decodedBuffer();
@@ -211,6 +259,9 @@ public class GridNioSslFilter extends GridNioFilterAdapter {
catch (SSLException e) {
U.error(log, "Failed to start SSL handshake (will close inbound connection): " + ses, e);
+ if (rejectedSesCnt != null)
+ rejectedSesCnt.increment();
+
ses.close();
}
}
@@ -358,6 +409,9 @@ public class GridNioSslFilter extends GridNioFilterAdapter {
}
}
catch (SSLException e) {
+ if (rejectedSesCnt != null)
+ rejectedSesCnt.increment();
+
throw new GridNioException("Failed to decode SSL data: " + ses, e);
}
finally {
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/GridNioServerWrapper.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/GridNioServerWrapper.java
index ae6f6d3..53c80a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/GridNioServerWrapper.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/GridNioServerWrapper.java
@@ -879,9 +879,12 @@ public class GridNioServerWrapper {
filters.add(new GridConnectionBytesVerifyFilter(log));
if (stateProvider.isSslEnabled()) {
- GridNioSslFilter sslFilter =
- new GridNioSslFilter(igniteCfg.getSslContextFactory().create(),
- true, ByteOrder.LITTLE_ENDIAN, log);
+ GridNioSslFilter sslFilter = new GridNioSslFilter(
+ igniteCfg.getSslContextFactory().create(),
+ true,
+ ByteOrder.LITTLE_ENDIAN,
+ log,
+ metricMgr == null ? null : metricMgr.registry(COMMUNICATION_METRICS_GROUP_NAME));
sslFilter.directMode(true);
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 9f59ded..7fe93d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -7021,10 +7021,13 @@ class ServerImpl extends TcpDiscoveryImpl {
if (log.isDebugEnabled())
U.error(log, "Caught exception on handshake [err=" + e + ", sock=" + sock + ']', e);
- if (X.hasCause(e, SSLException.class) && spi.isSslEnabled() && !spi.isNodeStopping0())
+ if (X.hasCause(e, SSLException.class) && spi.isSslEnabled() && !spi.isNodeStopping0()) {
LT.warn(log, "Failed to initialize connection " +
"(missing SSL configuration on remote node?) " +
"[rmtAddr=" + sock.getInetAddress() + ']', true);
+
+ spi.stats.onSslConnectionRejected();
+ }
else if ((X.hasCause(e, ObjectStreamException.class) || !sock.isClosed())
&& !spi.isNodeStopping0()) {
if (U.isMacInvalidArgumentError(e))
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 0f4fc15..4eea69d 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -1497,6 +1497,8 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements IgniteDiscovery
stats.registerMetrics(discoReg);
+ discoReg.register("SslEnabled", this::isSslEnabled, "Whether SSL is enabled.");
+
discoReg.register("MessageWorkerQueueSize", () -> impl.getMessageWorkerQueueSize(),
"Message worker queue current size");
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
index 27197a7..527347d 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryStatistics.java
@@ -73,6 +73,9 @@ public class TcpDiscoveryStatistics {
/** Pending messages registered count. */
private final IntMetricImpl pendingMsgsRegistered;
+ /** Metric that indicates connections count that were rejected due to SSL errors. */
+ private final IntMetricImpl rejectedSslConnectionsCnt;
+
/** */
public TcpDiscoveryStatistics() {
joinedNodesCnt = new IntMetricImpl(metricName(DISCO_METRICS, "JoinedNodes"), "Joined nodes count");
@@ -83,6 +86,11 @@ public class TcpDiscoveryStatistics {
pendingMsgsRegistered = new IntMetricImpl(metricName(DISCO_METRICS, "PendingMessagesRegistered"),
"Pending messages registered count");
+
+ rejectedSslConnectionsCnt = new IntMetricImpl(
+ metricName(DISCO_METRICS, "RejectedSslConnectionsCount"),
+ "TCP discovery connections count that were rejected due to SSL errors."
+ );
}
/**
@@ -97,6 +105,7 @@ public class TcpDiscoveryStatistics {
discoReg.register(failedNodesCnt);
discoReg.register(leftNodesCnt);
discoReg.register(pendingMsgsRegistered);
+ discoReg.register(rejectedSslConnectionsCnt);
}
/**
@@ -127,6 +136,11 @@ public class TcpDiscoveryStatistics {
crdSinceTs.compareAndSet(0, U.currentTimeMillis());
}
+ /** Increments connections count that were rejected due to SSL errors. */
+ public void onSslConnectionRejected() {
+ rejectedSslConnectionsCnt.increment();
+ }
+
/**
* Collects necessary stats for message received by SPI.
*
@@ -319,6 +333,7 @@ public class TcpDiscoveryStatistics {
procMsgs.clear();
rcvdMsgs.clear();
sentMsgs.clear();
+ rejectedSslConnectionsCnt.reset();
}
/** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/nio/GridNioSslSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/nio/GridNioSslSelfTest.java
index 88eda28..aef4a42 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/nio/GridNioSslSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/nio/GridNioSslSelfTest.java
@@ -72,7 +72,7 @@ public class GridNioSslSelfTest extends GridNioSelfTest {
.sendQueueLimit(0)
.filters(
new GridNioCodecFilter(parser, log, false),
- new GridNioSslFilter(sslCtx, true, ByteOrder.nativeOrder(), log));
+ new GridNioSslFilter(sslCtx, true, ByteOrder.nativeOrder(), log, null));
}
/** {@inheritDoc} */