You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by il...@apache.org on 2019/02/26 13:20:05 UTC

[ignite] branch master updated: IGNITE-6578 IGNITE_DIAGNOSTIC_WARN_LIMIT to control number of messages waiting on PME - Fixes #6168.

This is an automated email from the ASF dual-hosted git repository.

ilyak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git


The following commit(s) were added to refs/heads/master by this push:
     new 780946e  IGNITE-6578 IGNITE_DIAGNOSTIC_WARN_LIMIT to control number of messages waiting on PME - Fixes #6168.
780946e is described below

commit 780946e8cf1b467123cff1374b6b129a12d82403
Author: Roman Guseinov <gr...@gmail.com>
AuthorDate: Tue Feb 26 16:14:47 2019 +0300

    IGNITE-6578 IGNITE_DIAGNOSTIC_WARN_LIMIT to control number of messages waiting on PME - Fixes #6168.
    
    Signed-off-by: Ilya Kasnacheev <il...@gmail.com>
---
 .../org/apache/ignite/IgniteSystemProperties.java  |   5 +
 .../cache/GridCachePartitionExchangeManager.java   | 167 +++++++++--
 ...dCachePartitionExchangeManagerWarningsTest.java | 329 +++++++++++++++++++++
 .../ignite/testsuites/IgniteCacheTestSuite5.java   |   3 +
 4 files changed, 479 insertions(+), 25 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 5b9788f..ff58c60 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -1138,6 +1138,11 @@ public final class IgniteSystemProperties {
     public static final String IGNITE_BASELINE_AUTO_ADJUST_ENABLED = "IGNITE_BASELINE_AUTO_ADJUST_ENABLED";
 
     /**
+     * Maximum number of diagnostic warning messages per category, when waiting for PME.
+     */
+    public static final String IGNITE_DIAGNOSTIC_WARN_LIMIT = "IGNITE_DIAGNOSTIC_WARN_LIMIT";
+
+    /**
      * Enforces singleton.
      */
     private IgniteSystemProperties() {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 2568802..a6ebcc9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -124,6 +124,7 @@ import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_DIAGNOSTIC_WARN_LIMIT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_IO_DUMP_ON_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PRELOAD_RESEND_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_THREAD_DUMP_ON_EXCHANGE_TIMEOUT;
@@ -154,6 +155,9 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
         IgniteSystemProperties.getLong(IgniteSystemProperties.IGNITE_EXCHANGE_MERGE_DELAY, 0);
 
     /** */
+    private final int DIAGNOSTIC_WARN_LIMIT = IgniteSystemProperties.getInteger(IGNITE_DIAGNOSTIC_WARN_LIMIT, 10);
+
+    /** */
     private static final IgniteProductVersion EXCHANGE_PROTOCOL_2_SINCE = IgniteProductVersion.fromString("2.1.4");
 
     /** Atomic reference for pending partition resend timeout object. */
@@ -1856,15 +1860,20 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
         exchWorker.dumpExchangeDebugInfo();
 
         if (!readyFuts.isEmpty()) {
-            U.warn(diagnosticLog, "First 5 pending affinity ready futures [total=" + readyFuts.size() + ']');
+            int warningsLimit = IgniteSystemProperties.getInteger(IGNITE_DIAGNOSTIC_WARN_LIMIT, 5);
 
-            int cnt = 0;
+            U.warn(diagnosticLog, "First " + warningsLimit + " pending affinity ready futures [total=" +
+                readyFuts.size() + ']');
 
-            for (AffinityReadyFuture fut : readyFuts.values()) {
-                U.warn(diagnosticLog, ">>> " + fut);
+            if (warningsLimit > 0) {
+                int cnt = 0;
 
-                if (++cnt == 5)
-                    break;
+                for (AffinityReadyFuture fut : readyFuts.values()) {
+                    U.warn(diagnosticLog, ">>> " + fut);
+
+                    if (++cnt == warningsLimit)
+                        break;
+                }
             }
         }
 
@@ -1877,15 +1886,18 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
         ExchangeFutureSet exchFuts = this.exchFuts;
 
         if (exchFuts != null) {
-            U.warn(diagnosticLog, "Last 10 exchange futures (total: " + exchFuts.size() + "):");
+            U.warn(diagnosticLog, "Last " + DIAGNOSTIC_WARN_LIMIT + " exchange futures (total: " +
+                exchFuts.size() + "):");
 
-            int cnt = 0;
+            if (DIAGNOSTIC_WARN_LIMIT > 0) {
+                int cnt = 0;
 
-            for (GridDhtPartitionsExchangeFuture fut : exchFuts.values()) {
-                U.warn(diagnosticLog, ">>> " + fut.shortInfo());
+                for (GridDhtPartitionsExchangeFuture fut : exchFuts.values()) {
+                    U.warn(diagnosticLog, ">>> " + fut.shortInfo());
 
-                if (++cnt == 10)
-                    break;
+                    if (++cnt == DIAGNOSTIC_WARN_LIMIT)
+                        break;
+                }
             }
         }
 
@@ -1936,40 +1948,67 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
             new IgniteDiagnosticPrepareContext(cctx.localNodeId()) : null;
 
         if (tm != null) {
+            WarningsGroup warnings = new WarningsGroup("First %d long running transactions [total=%d]",
+                diagnosticLog, DIAGNOSTIC_WARN_LIMIT);
+
             for (IgniteInternalTx tx : tm.activeTransactions()) {
                 if (curTime - tx.startTime() > timeout) {
                     found = true;
 
-                    U.warn(diagnosticLog, "Found long running transaction [startTime=" + formatTime(tx.startTime()) +
-                        ", curTime=" + formatTime(curTime) + ", tx=" + tx + ']');
+                    if (warnings.canAddMessage()) {
+                        warnings.add(">>> Transaction [startTime=" + formatTime(tx.startTime()) +
+                            ", curTime=" + formatTime(curTime) + ", tx=" + tx + ']');
+                    }
+                    else
+                        warnings.incTotal();
                 }
             }
+
+            warnings.printToLog();
         }
 
         if (mvcc != null) {
+            WarningsGroup activeWarnings = new WarningsGroup("First %d long running cache futures [total=%d]",
+                diagnosticLog, DIAGNOSTIC_WARN_LIMIT);
+
             for (GridCacheFuture<?> fut : mvcc.activeFutures()) {
                 if (curTime - fut.startTime() > timeout) {
                     found = true;
 
-                    U.warn(diagnosticLog, "Found long running cache future [startTime=" + formatTime(fut.startTime()) +
-                        ", curTime=" + formatTime(curTime) + ", fut=" + fut + ']');
+                    if (activeWarnings.canAddMessage()) {
+                        activeWarnings.add(">>> Future [startTime=" + formatTime(fut.startTime()) +
+                            ", curTime=" + formatTime(curTime) + ", fut=" + fut + ']');
+                    }
+                    else
+                        activeWarnings.incTotal();
 
                     if (diagCtx != null && fut instanceof IgniteDiagnosticAware)
                         ((IgniteDiagnosticAware)fut).addDiagnosticRequest(diagCtx);
                 }
             }
 
+            activeWarnings.printToLog();
+
+            WarningsGroup atomicWarnings = new WarningsGroup("First %d long running cache futures [total=%d]",
+                diagnosticLog, DIAGNOSTIC_WARN_LIMIT);
+
             for (GridCacheFuture<?> fut : mvcc.atomicFutures()) {
                 if (curTime - fut.startTime() > timeout) {
                     found = true;
 
-                    U.warn(diagnosticLog, "Found long running cache future [startTime=" + formatTime(fut.startTime()) +
-                        ", curTime=" + formatTime(curTime) + ", fut=" + fut + ']');
+                    if (atomicWarnings.canAddMessage()) {
+                        atomicWarnings.add(">>> Future [startTime=" + formatTime(fut.startTime()) +
+                            ", curTime=" + formatTime(curTime) + ", fut=" + fut + ']');
+                    }
+                    else
+                        atomicWarnings.incTotal();
 
                     if (diagCtx != null && fut instanceof IgniteDiagnosticAware)
                         ((IgniteDiagnosticAware)fut).addDiagnosticRequest(diagCtx);
                 }
             }
+
+            atomicWarnings.printToLog();
         }
 
         if (diagCtx != null && !diagCtx.empty()) {
@@ -2706,16 +2745,18 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
          * Dump debug info.
          */
         void dumpExchangeDebugInfo() {
-            U.warn(log, "First 10 pending exchange futures [total=" + futQ.size() + ']');
+            U.warn(log, "First " + DIAGNOSTIC_WARN_LIMIT + " pending exchange futures [total=" + futQ.size() + ']');
 
-            int cnt = 0;
+            if (DIAGNOSTIC_WARN_LIMIT > 0) {
+                int cnt = 0;
 
-            for (CachePartitionExchangeWorkerTask task : futQ) {
-                if (task instanceof GridDhtPartitionsExchangeFuture) {
-                    U.warn(log, ">>> " + ((GridDhtPartitionsExchangeFuture)task).shortInfo());
+                for (CachePartitionExchangeWorkerTask task : futQ) {
+                    if (task instanceof GridDhtPartitionsExchangeFuture) {
+                        U.warn(log, ">>> " + ((GridDhtPartitionsExchangeFuture)task).shortInfo());
 
-                    if (++cnt == 10)
-                        break;
+                        if (++cnt == DIAGNOSTIC_WARN_LIMIT)
+                            break;
+                    }
                 }
             }
         }
@@ -3366,4 +3407,80 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
             return S.toString(AffinityReadyFuture.class, this, super.toString());
         }
     }
+
+    /**
+     * Class to print only limited number of warnings.
+     */
+    private static class WarningsGroup {
+        /** */
+        private final IgniteLogger log;
+
+        /** */
+        private final int warningsLimit;
+
+        /** */
+        private final String title;
+
+        /** */
+        private final List<String> messages = new ArrayList<>();
+
+        /** */
+        private int warningsTotal = 0;
+
+        /**
+         * @param log Target logger.
+         * @param warningsLimit Warnings limit.
+         */
+        private WarningsGroup(String title, IgniteLogger log, int warningsLimit) {
+            this.title = title;
+
+            this.log = log;
+
+            this.warningsLimit = warningsLimit;
+        }
+
+        /**
+         * @param msg Warning message.
+         * @return {@code true} if message is added to list.
+         */
+        private boolean add(String msg) {
+            boolean added = false;
+
+            if (canAddMessage()) {
+                messages.add(msg);
+
+                added = true;
+            }
+
+            warningsTotal++;
+
+            return added;
+        }
+
+        /**
+         * @return {@code true} if messages list size less than limit.
+         */
+        private boolean canAddMessage() {
+            return warningsTotal < warningsLimit;
+        }
+
+        /**
+         * Increase total number of warnings.
+         */
+        private void incTotal() {
+            warningsTotal++;
+        }
+
+        /**
+         * Print warnings block title and messages.
+         */
+        private void printToLog() {
+            if (warningsTotal > 0) {
+                U.warn(log, String.format(title, warningsLimit, warningsTotal));
+
+                for (String message : messages)
+                    U.warn(log, message);
+            }
+        }
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridCachePartitionExchangeManagerWarningsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridCachePartitionExchangeManagerWarningsTest.java
new file mode 100644
index 0000000..8f2560d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridCachePartitionExchangeManagerWarningsTest.java
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteTransactions;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicDeferredUpdateResponse;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.ListeningTestLogger;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+import org.jetbrains.annotations.Nullable;
+import org.junit.Test;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT;
+
+/**
+ * Test exchange manager warnings.
+ */
+public class GridCachePartitionExchangeManagerWarningsTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final String CACHE_NAME = "TEST_CACHE";
+
+    /** */
+    private String oldLongOpsDumpTimeout;
+
+    /** */
+    private CustomTestLogger testLog;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        oldLongOpsDumpTimeout = System.getProperty(IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        if (oldLongOpsDumpTimeout != null)
+            System.setProperty(IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT, oldLongOpsDumpTimeout);
+        else
+            System.clearProperty(IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        if (testLog != null)
+            testLog.clearListeners();
+
+        testLog = null;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @Test
+    public void testLongRunningCacheFutures() throws Exception {
+        long timeout = 1000;
+
+        System.setProperty(IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT, Long.toString(timeout));
+
+        testLog = new CustomTestLogger(false, log, "future");
+
+        int longRunFuturesCnt = 1000;
+
+        try (Ignite srv1 = start("srv-1", false, false)) {
+            try (Ignite srv2 = start("srv-2", false, false)) {
+                try (Ignite client = start("client", true, false)) {
+                    try (IgniteDataStreamer<Integer, Integer> streamer = client.dataStreamer(CACHE_NAME)) {
+                        streamer.allowOverwrite(true);
+
+                        for (int i = 0; i < longRunFuturesCnt; i++)
+                            streamer.addData(i, i);
+                    }
+
+                    Thread.sleep(timeout * 2);
+                }
+            }
+        }
+
+        assertTrue("Warnings were not found", testLog.warningsTotal() > 0);
+
+        assertTrue("Too much warnings in the logs: " + testLog.warningsTotal(),
+            testLog.warningsTotal() < longRunFuturesCnt);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @Test
+    public void testLongRunningTransactions() throws Exception {
+        long timeout = 1000;
+
+        System.setProperty(IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT, Long.toString(timeout));
+
+        testLog = new CustomTestLogger(false, log, "transaction");
+
+        int transactions = 100;
+
+        ExecutorService excSvc = Executors.newFixedThreadPool(transactions);
+
+        try (Ignite srv1 = start("srv", false, true)) {
+            CountDownLatch txStarted = new CountDownLatch(transactions);
+
+            CountDownLatch stopTx = new CountDownLatch(1);
+
+            for (int i = 0; i < transactions; i++)
+                excSvc.submit(new AsyncTransaction(srv1, CACHE_NAME, i, txStarted, stopTx));
+
+            if (!txStarted.await(10000, TimeUnit.MILLISECONDS))
+                fail("Unable to start transactions");
+
+            Thread.sleep(timeout * 2);
+
+            stopTx.countDown();
+        }
+        finally {
+            excSvc.shutdown();
+
+            if (!excSvc.awaitTermination(10000, TimeUnit.MILLISECONDS))
+                fail("Unable to wait for thread pool termination.");
+        }
+
+        assertTrue("Warnings were not found", testLog.warningsTotal() > 0);
+
+        assertTrue("Too much warnings in the logs: " + testLog.warningsTotal(),
+            testLog.warningsTotal() < transactions);
+    }
+
+    /**
+     * Start Ignite node.
+     *
+     * @param instanceName Ignite instance name.
+     * @param clientMode Client mode flag.
+     * @param transactional Transactional cache flag.
+     * @throws Exception If failed.
+     */
+    private Ignite start(String instanceName, boolean clientMode, boolean transactional) throws Exception {
+        return Ignition.start(getConfiguration(instanceName, clientMode, transactional));
+    }
+
+    /**
+     * Create Ignite configuration.
+     *
+     * @param instanceName Ignite instance name.
+     * @param clientMode Client mode flag.
+     * @param transactional Transactional cache flag.
+     * @throws Exception If failed.
+     */
+    private IgniteConfiguration getConfiguration(String instanceName, boolean clientMode, boolean transactional) throws Exception {
+        CacheConfiguration cacheCfg = new CacheConfiguration(CACHE_NAME)
+            .setBackups(1)
+            .setAtomicityMode(transactional ? CacheAtomicityMode.TRANSACTIONAL : CacheAtomicityMode.ATOMIC);
+
+        IgniteConfiguration cfg = getConfiguration()
+            .setCacheConfiguration(cacheCfg)
+            .setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER))
+            .setCommunicationSpi(new CustomTcpCommunicationSpi())
+            .setIgniteInstanceName(instanceName)
+            .setClientMode(clientMode);
+
+        if (testLog != null)
+            cfg.setGridLogger(testLog);
+
+        return cfg;
+    }
+
+    /**
+     * Async tx runnable.
+     */
+    private static class AsyncTransaction implements Runnable {
+        /** */
+        private final Ignite ignite;
+
+        /** */
+        private final String cacheName;
+
+        /** */
+        private final Integer key;
+
+        /** */
+        private final CountDownLatch startLatch;
+
+        /** */
+        private final CountDownLatch canStopLatch;
+
+        /**
+         * @param ignite Ignite instance.
+         * @param cacheName Cache name.
+         * @param key Cache key.
+         * @param startLatch Latch to synchronize all transactions.
+         * @param canStopLatch Latch to finish transaction.
+         */
+        public AsyncTransaction(Ignite ignite, String cacheName, Integer key, CountDownLatch startLatch, CountDownLatch canStopLatch) {
+            this.ignite = ignite;
+            this.cacheName = cacheName;
+            this.key = key;
+            this.startLatch = startLatch;
+            this.canStopLatch = canStopLatch;
+        }
+
+        /**
+         * Start transaction.
+         */
+        @Override public void run() {
+            IgniteTransactions transactions = ignite.transactions();
+
+            try (Transaction tx = transactions.txStart(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.SERIALIZABLE)) {
+                IgniteCache<Integer, Integer> cache = ignite.cache(cacheName);
+
+                Integer val = cache.get(key);
+
+                startLatch.countDown();
+
+                cache.put(key, val == null ? 1 : val + 1);
+
+                try {
+                    canStopLatch.await();
+                }
+                catch (InterruptedException e) {
+                    e.printStackTrace();
+
+                    Thread.currentThread().interrupt();
+                }
+
+                tx.rollback();
+            }
+        }
+    }
+
+    /**
+     * Custom logger for counting warning messages.
+     */
+    private static class CustomTestLogger extends ListeningTestLogger {
+        /** */
+        private final AtomicInteger warningsTotal = new AtomicInteger();
+
+        /** */
+        private final String substr;
+
+        /**
+         * @param dbg If set to {@code true}, enables debug and trace log messages processing.
+         * @param echo Logger to echo all messages, limited by {@code dbg} flag.
+         * @param substr Substring to filter warning messages.
+         */
+        public CustomTestLogger(boolean dbg, @Nullable IgniteLogger echo, String substr) {
+            super(dbg, echo);
+
+            this.substr = substr;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void warning(String msg, @Nullable Throwable t) {
+            super.warning(msg, t);
+
+            if (substr == null || msg.toLowerCase().contains(substr.toLowerCase()))
+                warningsTotal.incrementAndGet();
+        }
+
+        /**
+         * @return Total number of warnings.
+         */
+        public int warningsTotal() {
+            return warningsTotal.get();
+        }
+    }
+
+    /**
+     * Custom communication SPI for simulating long running cache futures.
+     */
+    private static class CustomTcpCommunicationSpi extends TcpCommunicationSpi {
+        /** {@inheritDoc} */
+        public CustomTcpCommunicationSpi() {
+            // No-op.
+        }
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg,
+            IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+            if (msg instanceof GridIoMessage) {
+                // Skip response from backup node.
+                if (((GridIoMessage)msg).message() instanceof GridDhtAtomicDeferredUpdateResponse)
+                    return;
+            }
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
index 5bc1c71..e58b38d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
@@ -27,6 +27,7 @@ import org.apache.ignite.cache.affinity.AffinityDistributionLoggingTest;
 import org.apache.ignite.cache.affinity.AffinityHistoryCleanupTest;
 import org.apache.ignite.cache.affinity.local.LocalAffinityFunctionTest;
 import org.apache.ignite.internal.GridCachePartitionExchangeManagerHistSizeTest;
+import org.apache.ignite.internal.GridCachePartitionExchangeManagerWarningsTest;
 import org.apache.ignite.internal.processors.cache.CacheKeepBinaryTransactionTest;
 import org.apache.ignite.internal.processors.cache.CacheNearReaderUpdateTest;
 import org.apache.ignite.internal.processors.cache.CacheRebalancingSelfTest;
@@ -111,6 +112,8 @@ public class IgniteCacheTestSuite5 {
 
         GridTestUtils.addTestIfNeeded(suite, GridCachePartitionExchangeManagerHistSizeTest.class, ignoredTests);
 
+        GridTestUtils.addTestIfNeeded(suite, GridCachePartitionExchangeManagerWarningsTest.class, ignoredTests);
+
         GridTestUtils.addTestIfNeeded(suite, NotMappedPartitionInTxTest.class, ignoredTests);
 
         GridTestUtils.addTestIfNeeded(suite, ConcurrentCacheStartTest.class, ignoredTests);