You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sd...@apache.org on 2022/06/14 12:01:28 UTC
[ignite] branch master updated: IGNITE-17152 Improve logging levels for situations when dealing with a client node (#10085)
This is an automated email from the ASF dual-hosted git repository.
sdanilov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 270d7f83879 IGNITE-17152 Improve logging levels for situations when dealing with a client node (#10085)
270d7f83879 is described below
commit 270d7f838791909e6645af853127c2af8dbc8e0a
Author: Roman Puchkovskiy <ro...@gmail.com>
AuthorDate: Tue Jun 14 16:01:21 2022 +0400
IGNITE-17152 Improve logging levels for situations when dealing with a client node (#10085)
---
.../communication/tcp/ClientExceptionsUtils.java | 57 ++++++
.../spi/communication/tcp/TcpCommunicationSpi.java | 15 +-
.../tcp/internal/CommunicationWorker.java | 2 +-
.../tcp/ClientExceptionsUtilsTest.java | 87 +++++++++
.../tcp/CommunicationWorkerThreadUtils.java | 49 +++++
...unicationInverseConnectionEstablishingTest.java | 14 +-
...mmunicationSpiInverseConnectionLoggingTest.java | 209 +++++++++++++++++++++
.../TcpCommunicationSpiNodeLeftLoggingTest.java | 152 +++++++++++++++
.../ignite/testframework/MemorizingAppender.java | 102 ++++++++++
.../testframework/MemorizingAppenderTest.java | 69 +++++++
.../ignite/testsuites/IgniteBasicTestSuite2.java | 3 +
.../IgniteSpiCommunicationSelfTestSuite.java | 7 +
.../query/h2/twostep/GridMapQueryExecutor.java | 9 +-
13 files changed, 756 insertions(+), 19 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/ClientExceptionsUtils.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/ClientExceptionsUtils.java
new file mode 100644
index 00000000000..f00b80624ec
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/ClientExceptionsUtils.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp;
+
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.spi.IgniteSpiException;
+
+/**
+ * Utils to analyze client-related exceptions.
+ */
+public class ClientExceptionsUtils {
+ /**
+ * Returns {@code true} if the exception relates to cluster topology change that prevents a connection, AND the given node is client.
+ *
+ * @param t The exception we analyze.
+ * @param node Node to which we tried to send a message, but the send produced the given exception.
+ * @return {@code true} if the exception relates to cluster topology change that prevents a connection, AND the given node is client.
+ */
+ public static boolean isClientNodeTopologyException(Throwable t, ClusterNode node) {
+ ClusterTopologyCheckedException ex = X.cause(t, ClusterTopologyCheckedException.class);
+
+ return ex != null && node.isClient();
+ }
+
+ /**
+ * Returns {@code true} if the exception that is provided is thrown because an attempt to open a direct connection
+ * was made while only inverse connections are allowed.
+ *
+ * @param t Exception to inspect.
+ * @return {@code true} if the exception that is provided is thrown because an attempt to open a direct connection
+ * was made while only inverse connections are allowed.
+ */
+ public static boolean isAttemptToEstablishDirectConnectionWhenOnlyInverseIsAllowed(Throwable t) {
+ IgniteSpiException igniteSpiException = X.cause(t, IgniteSpiException.class);
+
+ return igniteSpiException != null && igniteSpiException.getMessage() != null
+ && igniteSpiException.getMessage().contains(
+ "because it is started in 'forceClientToServerConnections' mode; inverse connection will be requested");
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 6d0990c77f1..81105b23558 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -1131,10 +1131,17 @@ public class TcpCommunicationSpi extends TcpCommunicationConfigInitializer {
if (stopping)
throw new IgniteSpiException("Node is stopping.", t);
- // NodeUnreachableException should not be explicitly logged. Error message will appear if inverse
- // connection attempt fails as well.
- if (!(t instanceof NodeUnreachableException))
- log.error("Failed to send message to remote node [node=" + node + ", msg=" + msg + ']', t);
+ String messageForLog = "Failed to send message to remote node [node=" + node + ", msg=" + msg + ']';
+
+ if (ClientExceptionsUtils.isClientNodeTopologyException(t, node)
+ || ClientExceptionsUtils.isAttemptToEstablishDirectConnectionWhenOnlyInverseIsAllowed(t))
+ log.warning(messageForLog, t);
+ else {
+ // NodeUnreachableException should not be explicitly logged. Error message will appear if inverse
+ // connection attempt fails as well.
+ if (!(t instanceof NodeUnreachableException))
+ log.error(messageForLog, t);
+ }
if (t instanceof Error)
throw (Error)t;
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/CommunicationWorker.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/CommunicationWorker.java
index 77ead1506f4..2be20ea5b7d 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/CommunicationWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/CommunicationWorker.java
@@ -56,7 +56,7 @@ import static org.apache.ignite.spi.communication.tcp.internal.CommunicationTcpU
*/
public class CommunicationWorker extends GridWorker {
/** Worker name. */
- private static final String WORKER_NAME = "tcp-comm-worker";
+ public static final String WORKER_NAME = "tcp-comm-worker";
/** Config. */
private final TcpCommunicationConfiguration cfg;
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/ClientExceptionsUtilsTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/ClientExceptionsUtilsTest.java
new file mode 100644
index 00000000000..c29d1897fce
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/ClientExceptionsUtilsTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.lenient;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests of {@link ClientExceptionsUtils}.
+ */
+@RunWith(MockitoJUnitRunner.class)
+public class ClientExceptionsUtilsTest {
+ /***/
+ @Mock
+ private ClusterNode node;
+
+ /**
+ * Tests that {@link ClientExceptionsUtils#isClientNodeTopologyException(Throwable, ClusterNode)} detects
+ * client {@link ClusterTopologyCheckedException}.
+ */
+ @Test
+ public void detectsClientNodeTopologyException() {
+ when(node.isClient()).thenReturn(true);
+
+ assertTrue(ClientExceptionsUtils.isClientNodeTopologyException(clusterTopologyCheckedException(), node));
+ }
+
+ /**
+ * Tests that {@link ClientExceptionsUtils#isClientNodeTopologyException(Throwable, ClusterNode)} returns {@code false}
+ * when node is not a client.
+ */
+ @Test
+ public void doesNotDetectsClientNodeTopologyExceptionForNonClient() {
+ when(node.isClient()).thenReturn(false);
+
+ assertFalse(ClientExceptionsUtils.isClientNodeTopologyException(clusterTopologyCheckedException(), node));
+ }
+
+ /**
+ * Tests that {@link ClientExceptionsUtils#isClientNodeTopologyException(Throwable, ClusterNode)} returns {@code false}
+ * when exception is not a {@link ClusterTopologyCheckedException}.
+ */
+ @Test
+ public void doesNotDetectClientNodeTopologyExceptionForOtherExceptions() {
+ lenient().when(node.isClient()).thenReturn(true);
+
+ assertFalse(ClientExceptionsUtils.isClientNodeTopologyException(new IgniteCheckedException(), node));
+ }
+
+ /**
+ * Returns a {@link ClusterTopologyCheckedException} that is produced when an attempt to await for an inverse connection
+ * fails due to a timeout.
+ *
+ * @return A {@link ClusterTopologyCheckedException} that is produced when an attempt to await for an inverse connection
+ * fails due to a timeout.
+ */
+ private Exception clusterTopologyCheckedException() {
+ return new ClusterTopologyCheckedException(
+ "Failed to wait for establishing inverse connection (node left topology): 67cf0e5e-974c-463a-a1f2-915fe3cdd3e7"
+ );
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/CommunicationWorkerThreadUtils.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/CommunicationWorkerThreadUtils.java
new file mode 100644
index 00000000000..a95b8ddb6d6
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/CommunicationWorkerThreadUtils.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.communication.tcp.internal.CommunicationWorker;
+
+/**
+ * Utils to work with communication worker threads.
+ */
+class CommunicationWorkerThreadUtils {
+ /**
+ * We need to interrupt communication worker client nodes so that
+ * closed connection won't automatically reopen when we don't expect it.
+ *
+ * @param clientName The name of the client whose threads we want to interrupt.
+ * @param log The logger to use while joining the interrupted threads.
+ */
+ static void interruptCommWorkerThreads(String clientName, IgniteLogger log) {
+ List<Thread> tcpCommWorkerThreads = Thread.getAllStackTraces().keySet().stream()
+ .filter(t -> t.getName().contains(CommunicationWorker.WORKER_NAME))
+ .filter(t -> t.getName().contains(clientName))
+ .collect(Collectors.toList());
+
+ for (Thread tcpCommWorkerThread : tcpCommWorkerThreads) {
+ U.interrupt(tcpCommWorkerThread);
+
+ U.join(tcpCommWorkerThread, log);
+ }
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationInverseConnectionEstablishingTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationInverseConnectionEstablishingTest.java
index 4e61befa90a..a270a415344 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationInverseConnectionEstablishingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationInverseConnectionEstablishingTest.java
@@ -19,13 +19,11 @@ package org.apache.ignite.spi.communication.tcp;
import java.util.Collections;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.UnaryOperator;
-import java.util.stream.Collectors;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
@@ -42,7 +40,6 @@ import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.managers.communication.GridIoMessage;
import org.apache.ignite.internal.util.nio.GridCommunicationClient;
-import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.plugin.extensions.communication.Message;
@@ -324,16 +321,7 @@ public class GridTcpCommunicationInverseConnectionEstablishingTest extends GridC
* closed connection won't automatically reopen when we don't expect it.
*/
private void interruptCommWorkerThreads(String clientName) {
- List<Thread> tcpCommWorkerThreads = Thread.getAllStackTraces().keySet().stream()
- .filter(t -> t.getName().contains("tcp-comm-worker"))
- .filter(t -> t.getName().contains(clientName))
- .collect(Collectors.toList());
-
- for (Thread tcpCommWorkerThread : tcpCommWorkerThreads) {
- U.interrupt(tcpCommWorkerThread);
-
- U.join(tcpCommWorkerThread, log);
- }
+ CommunicationWorkerThreadUtils.interruptCommWorkerThreads(clientName, log);
}
/**
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiInverseConnectionLoggingTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiInverseConnectionLoggingTest.java
new file mode 100644
index 00000000000..4893b4c3e3c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiInverseConnectionLoggingTest.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.util.UUIDCollectionMessage;
+import org.apache.ignite.internal.util.nio.GridCommunicationClient;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.internal.TcpInverseConnectionResponseMessage;
+import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.MemorizingAppender;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.log4j.Level;
+import org.apache.log4j.spi.LoggingEvent;
+import org.junit.Test;
+
+import static java.util.Collections.singletonList;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+/**
+ * Tests logging in {@link TcpCommunicationSpi} when inverse connection to a client is tried to be established.
+ */
+public class TcpCommunicationSpiInverseConnectionLoggingTest extends GridCommonAbstractTest {
+ /***/
+ private static final String SERVER_NAME = "server";
+
+ /***/
+ private static final String CLIENT_NAME = "client";
+
+ /** */
+ private static final String UNREACHABLE_IP = "172.31.30.132";
+
+ /***/
+ private final MemorizingAppender log4jAppender = new MemorizingAppender();
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setFailureDetectionTimeout(2_000);
+
+ TcpCommunicationSpi spi = new TestCommunicationSpi();
+
+ spi.setForceClientToServerConnections(true);
+
+ cfg.setCommunicationSpi(spi);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ log4jAppender.installSelfOn(TestCommunicationSpi.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ log4jAppender.removeSelfFrom(TestCommunicationSpi.class);
+
+ stopAllGrids();
+
+ super.afterTest();
+ }
+
+ /**
+ * Tests that an exception that is produced when we cause an inversion connection opening from the client's side
+ * is logged with WARN.
+ *
+ * @throws Exception If something goes wrong.
+ */
+ @Test
+ public void logsWarnForExceptionMeaningSwitchToInverseConnection() throws Exception {
+ IgniteEx server = startGrid(SERVER_NAME);
+ IgniteEx client = startClientGrid(CLIENT_NAME);
+
+ ClusterNode clientNode = client.localNode();
+
+ interruptCommWorkerThreads(client.name());
+
+ TcpCommunicationSpi spi = (TcpCommunicationSpi)server.configuration().getCommunicationSpi();
+
+ GridTestUtils.invoke(spi, "onNodeLeft", clientNode.consistentId(), clientNode.id());
+
+ sendFailingMessage(server, clientNode);
+
+ LoggingEvent event = log4jAppender.singleEventSatisfying(
+ evt -> evt.getRenderedMessage().startsWith("Failed to send message to remote node ")
+ );
+
+ assertThat(event.getLevel(), is(Level.WARN));
+ }
+
+ /**
+ * We need to interrupt communication worker client nodes so that
+ * closed connection won't automatically reopen when we don't expect it.
+ *
+ * @param clientName The name of the client whose threads we want to interrupt.
+ */
+ private void interruptCommWorkerThreads(String clientName) {
+ CommunicationWorkerThreadUtils.interruptCommWorkerThreads(clientName, log);
+ }
+
+ /**
+ * Sends some message from one Ignite node to another node, the send will fail because an inverse connection
+ * cannot be established.
+ *
+ * @param sourceIgnite Ignite node from which to send a message.
+ * @param targetNode Target node to which to send the message.
+ */
+ private void sendFailingMessage(Ignite sourceIgnite, ClusterNode targetNode) {
+ GridTestUtils.assertThrows(
+ log,
+ () -> sourceIgnite.configuration().getCommunicationSpi().sendMessage(targetNode, someMessage()),
+ Exception.class,
+ null
+ );
+ }
+
+ /**
+ * Returns some message.
+ *
+ * @return Some message.
+ */
+ private UUIDCollectionMessage someMessage() {
+ return new UUIDCollectionMessage(singletonList(UUID.randomUUID()));
+ }
+
+ /**
+ * A custom {@link TcpCommunicationSpi} that allows to model the situation when an inverse connection must be
+ * established, but it cannot be.
+ */
+ private static class TestCommunicationSpi extends TcpCommunicationSpi {
+ /** {@inheritDoc} */
+ @Override protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx) throws IgniteCheckedException {
+ if (node.isClient()) {
+ Map<String, Object> attrs = new HashMap<>(node.attributes());
+
+ attrs.put(createAttributeName(ATTR_ADDRS), Collections.singleton(UNREACHABLE_IP));
+ attrs.put(createAttributeName(ATTR_PORT), 47200);
+ attrs.put(createAttributeName(ATTR_EXT_ADDRS), Collections.emptyList());
+ attrs.put(createAttributeName(ATTR_HOST_NAMES), Collections.emptyList());
+
+ ((TcpDiscoveryNode)(node)).setAttributes(attrs);
+ }
+
+ return super.createTcpClient(node, connIdx);
+ }
+
+ /**
+ * Creates an attribute name by prepending it with the class name (and a dot).
+ *
+ * @param name Name.
+ */
+ private String createAttributeName(String name) {
+ return getClass().getSimpleName() + '.' + name;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void sendMessage(
+ ClusterNode node,
+ Message msg,
+ IgniteInClosure<IgniteException> ackC
+ ) throws IgniteSpiException {
+ if (msg instanceof GridIoMessage) {
+ GridIoMessage msg0 = (GridIoMessage)msg;
+
+ if (msg0.message() instanceof TcpInverseConnectionResponseMessage) {
+ if (log.isInfoEnabled())
+ log.info("Client skips inverse connection response to server: " + node);
+
+ return;
+ }
+ }
+
+ super.sendMessage(node, msg, ackC);
+ }
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiNodeLeftLoggingTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiNodeLeftLoggingTest.java
new file mode 100644
index 00000000000..4d30caa7731
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiNodeLeftLoggingTest.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp;
+
+import java.util.UUID;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.UUIDCollectionMessage;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.MemorizingAppender;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.log4j.Level;
+import org.apache.log4j.spi.LoggingEvent;
+import org.junit.Test;
+
+import static java.util.Collections.singletonList;
+import static org.apache.ignite.cluster.ClusterState.ACTIVE;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+/**
+ * Tests logging in {@link TcpCommunicationSpi} in relation to 'node left' event.
+ */
+public class TcpCommunicationSpiNodeLeftLoggingTest extends GridCommonAbstractTest {
+ /***/
+ private static final String SERVER1_NAME = "server1";
+
+ /***/
+ private static final String CLIENT_NAME = "client";
+
+ /***/
+ private final MemorizingAppender log4jAppender = new MemorizingAppender();
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ TcpCommunicationSpi spi = new TcpCommunicationSpi();
+
+ cfg.setCommunicationSpi(spi);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ log4jAppender.installSelfOn(TcpCommunicationSpi.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ log4jAppender.removeSelfFrom(TcpCommunicationSpi.class);
+
+ stopAllGrids();
+
+ super.afterTest();
+ }
+
+ /**
+ * Tests that when we cannot send a message to a server node that left the topology, then we log this at INFO level.
+ *
+ * @throws Exception If something goes wrong.
+ */
+ @Test
+ public void logsWithErrorWhenCantSendMessageToServerWhichLeft() throws Exception {
+ IgniteEx server1 = startGrid(SERVER1_NAME);
+ IgniteEx server2 = startGrid("server2");
+
+ ClusterNode server2Node = server2.localNode();
+
+ server1.cluster().state(ACTIVE);
+
+ server2.close();
+
+ sendFailingMessage(server1, server2Node);
+
+ LoggingEvent event = log4jAppender.singleEventSatisfying(
+ evt -> evt.getRenderedMessage().startsWith("Failed to send message to remote node")
+ );
+
+ assertThat(event.getLevel(), is(Level.ERROR));
+ }
+
+ /**
+ * Sends some message from one Ignite node to another node, the send will fail because the target node
+ * has already left.
+ *
+ * @param sourceIgnite Ignite node from which to send a message.
+ * @param targetNode Target node to which to send the message.
+ */
+ private void sendFailingMessage(Ignite sourceIgnite, ClusterNode targetNode) {
+ GridTestUtils.assertThrows(
+ log,
+ () -> sourceIgnite.configuration().getCommunicationSpi().sendMessage(targetNode, someMessage()),
+ Exception.class,
+ null
+ );
+ }
+
+ /**
+ * Returns some message.
+ *
+ * @return Some message.
+ */
+ private UUIDCollectionMessage someMessage() {
+ return new UUIDCollectionMessage(singletonList(UUID.randomUUID()));
+ }
+
+ /**
+ * Tests that when we cannot send a message to a client node that left the topology, then we log this at WARN level.
+ *
+ * @throws Exception If something goes wrong.
+ */
+ @Test
+ public void logsWithWarnWhenCantSendMessageToClientWhichLeft() throws Exception {
+ IgniteEx server = startGrid(SERVER1_NAME);
+ IgniteEx client = startClientGrid(CLIENT_NAME);
+
+ ClusterNode clientNode = client.localNode();
+
+ server.cluster().state(ACTIVE);
+
+ client.close();
+
+ sendFailingMessage(server, clientNode);
+
+ LoggingEvent event = log4jAppender.singleEventSatisfying(
+ evt -> evt.getRenderedMessage().startsWith("Failed to send message to remote node")
+ );
+
+ assertThat(event.getLevel(), is(Level.WARN));
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/MemorizingAppender.java b/modules/core/src/test/java/org/apache/ignite/testframework/MemorizingAppender.java
new file mode 100644
index 00000000000..a1d78c19328
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/MemorizingAppender.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testframework;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.function.Predicate;
+import org.apache.log4j.AppenderSkeleton;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
+
+import static java.util.stream.Collectors.toList;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.hasSize;
+
+/**
+ * A Log4j {@link org.apache.log4j.Appender} that memorizes all the events it gets from loggers. These events are made
+ * available to the class users.
+ */
+public class MemorizingAppender extends AppenderSkeleton {
+ /**
+ * Events that were seen by this Appender.
+ */
+ private final List<LoggingEvent> events = new CopyOnWriteArrayList<>();
+
+ /** {@inheritDoc} */
+ @Override protected void append(LoggingEvent event) {
+ events.add(event);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() {
+ // no-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean requiresLayout() {
+ return false;
+ }
+
+ /**
+ * Returns all events that were seen by this Appender so far.
+ *
+ * @return All events that were seen by this Appender so far.
+ */
+ public List<LoggingEvent> events() {
+ return new ArrayList<>(events);
+ }
+
+ /**
+ * Adds this Appender to the logger corresponding to the provided class.
+ *
+ * @param target Class on whose logger to install this Appender.
+ */
+ public void installSelfOn(Class<?> target) {
+ Logger logger = Logger.getLogger(target);
+
+ logger.addAppender(this);
+ }
+
+ /**
+ * Removes this Appender from the logger corresponding to the provided class.
+ *
+ * @param target Class from whose logger to remove this Appender.
+ */
+ public void removeSelfFrom(Class<?> target) {
+ Logger logger = Logger.getLogger(target);
+
+ logger.removeAppender(this);
+ }
+
+ /**
+ * Returns the single event satisfying the given predicate. If no such event exists or more than one event matches,
+ * then an exception is thrown.
+ *
+ * @param predicate Predicate to use to select the event.
+ * @return The single event satisfying the given predicate.
+ */
+ public LoggingEvent singleEventSatisfying(Predicate<LoggingEvent> predicate) {
+ List<LoggingEvent> matches = events.stream().filter(predicate).collect(toList());
+
+ assertThat(matches, hasSize(1));
+
+ return matches.get(0);
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/MemorizingAppenderTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/MemorizingAppenderTest.java
new file mode 100644
index 00000000000..c32e1232d06
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/MemorizingAppenderTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testframework;
+
+import java.util.List;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+
+/**
+ * Tests for {@link MemorizingAppender}.
+ */
+public class MemorizingAppenderTest {
+ /**
+ * The instance under test.
+ */
+ private final MemorizingAppender appender = new MemorizingAppender();
+
+ /***/
+ @Before
+ public void installAppender() {
+ appender.installSelfOn(MemorizingAppenderTest.class);
+ }
+
+ /***/
+ @After
+ public void removeAppender() {
+ appender.removeSelfFrom(MemorizingAppenderTest.class);
+ }
+
+ /**
+ * Tests that MemorizingAppender memorizes logging events.
+ */
+ @Test
+ public void memorizesLoggingEvents() {
+ Logger.getLogger(MemorizingAppenderTest.class).info("Hello!");
+
+ List<LoggingEvent> events = appender.events();
+
+ assertThat(events, hasSize(1));
+
+ LoggingEvent event = events.get(0);
+
+ assertThat(event.getLevel(), is(Level.INFO));
+ assertThat(event.getRenderedMessage(), is("Hello!"));
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java
index a87a042ea05..ffdf0ce643c 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite2.java
@@ -90,6 +90,7 @@ import org.apache.ignite.plugin.PluginNodeValidationTest;
import org.apache.ignite.plugin.security.SecurityPermissionSetBuilderTest;
import org.apache.ignite.spi.checkpoint.noop.NoopCheckpointSpiLoggingTest;
import org.apache.ignite.startup.properties.NotStringSystemPropertyTest;
+import org.apache.ignite.testframework.MemorizingAppenderTest;
import org.apache.ignite.testframework.MessageOrderLogListenerTest;
import org.apache.ignite.testframework.test.ConfigVariationsExecutionTest;
import org.apache.ignite.testframework.test.ConfigVariationsTestSuiteBuilderTest;
@@ -172,6 +173,8 @@ import org.junit.runners.Suite;
MessageOrderLogListenerTest.class,
+ MemorizingAppenderTest.class,
+
CacheLocalGetSerializationTest.class,
PluginNodeValidationTest.class,
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
index f51663e36be..be98eec6e00 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
@@ -17,6 +17,7 @@
package org.apache.ignite.testsuites;
+import org.apache.ignite.spi.communication.tcp.ClientExceptionsUtilsTest;
import org.apache.ignite.spi.communication.tcp.GridCacheDhtLockBackupSelfTest;
import org.apache.ignite.spi.communication.tcp.GridSandboxedClientWithoutNetworkTest;
import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationInverseConnectionEstablishingTest;
@@ -47,7 +48,9 @@ import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiFaultyClientSs
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiFaultyClientTest;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiFreezingClientTest;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiHalfOpenedConnectionTest;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiInverseConnectionLoggingTest;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiMultiJvmTest;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiNodeLeftLoggingTest;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiSkipMessageSendTest;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationStatisticsTest;
import org.apache.ignite.spi.communication.tcp.TooManyOpenFilesTcpCommunicationSpiTest;
@@ -109,6 +112,10 @@ import org.junit.runners.Suite;
GridCacheDhtLockBackupSelfTest.class,
TcpCommunicationHandshakeTimeoutTest.class,
+
+ TcpCommunicationSpiNodeLeftLoggingTest.class,
+ TcpCommunicationSpiInverseConnectionLoggingTest.class,
+ ClientExceptionsUtilsTest.class
})
public class IgniteSpiCommunicationSelfTestSuite {
}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index 730caf3f7cc..27f2b72e324 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -783,7 +783,14 @@ public class GridMapQueryExecutor {
catch (Exception e) {
e.addSuppressed(err);
- U.error(log, "Failed to send error message.", e);
+ String messageForLog = "Failed to send error message";
+
+ if (node.isClient()) {
+ if (log.isDebugEnabled())
+ log.debug(messageForLog + U.nl() + X.getFullStackTrace(e));
+ }
+ else
+ U.error(log, messageForLog, e);
}
}