You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2018/11/29 08:27:19 UTC

ignite git commit: IGNITE-8640 Fixed dynamic cache creation failures - Fixes #4480.

Repository: ignite
Updated Branches:
  refs/heads/master 2fca099fd -> 560240f14


IGNITE-8640 Fixed dynamic cache creation failures - Fixes #4480.

Signed-off-by: Alexey Goncharuk <al...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/560240f1
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/560240f1
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/560240f1

Branch: refs/heads/master
Commit: 560240f145fc40464a872d74f6c0bbdf950abd4a
Parents: 2fca099
Author: d.garus <ga...@gmail.com>
Authored: Thu Nov 29 11:25:59 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Thu Nov 29 11:25:59 2018 +0300

----------------------------------------------------------------------
 .../cache/CacheAffinitySharedManager.java       |   16 +-
 ...IgniteAbstractDynamicCacheStartFailTest.java | 1959 +++++++++++-------
 .../CacheMvccConfigurationValidationTest.java   |   37 +-
 .../query/SqlIllegalSchemaSelfTest.java         |  176 +-
 .../FailureHandlerTest.cs                       |    2 +
 5 files changed, 1354 insertions(+), 836 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/560240f1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 881f8db..ce87fa9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -924,14 +924,18 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
 
         Map<StartCacheInfo, IgniteCheckedException> failedCaches = cctx.cache().prepareStartCachesIfPossible(startCacheInfos.keySet());
 
-        failedCaches.forEach((cacheInfo, exception) -> {
-            U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " +
-                "[cacheName=" + cacheInfo.getStartedConfiguration().getName() + ']', exception);
+        for (Map.Entry<StartCacheInfo, IgniteCheckedException> entry : failedCaches.entrySet()) {
+            if (cctx.localNode().isClient()) {
+                U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " +
+                    "[cacheName=" + entry.getKey().getStartedConfiguration().getName() + ']', entry.getValue());
 
-            cctx.cache().closeCaches(Collections.singleton(cacheInfo.getStartedConfiguration().getName()), false);
+                cctx.cache().closeCaches(Collections.singleton(entry.getKey().getStartedConfiguration().getName()), false);
 
-            cctx.cache().completeCacheStartFuture(startCacheInfos.get(cacheInfo), false, exception);
-        });
+                cctx.cache().completeCacheStartFuture(startCacheInfos.get(entry.getKey()), false, entry.getValue());
+            }
+            else
+                throw entry.getValue();
+        }
 
         Set<StartCacheInfo> failedCacheInfos = failedCaches.keySet();
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/560240f1/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteAbstractDynamicCacheStartFailTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteAbstractDynamicCacheStartFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteAbstractDynamicCacheStartFailTest.java
index db13c11..d506224 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteAbstractDynamicCacheStartFailTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteAbstractDynamicCacheStartFailTest.java
@@ -1,786 +1,1173 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicInteger;
-import javax.cache.CacheException;
-import javax.cache.configuration.Factory;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.affinity.AffinityFunctionContext;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cache.query.annotations.QuerySqlField;
-import org.apache.ignite.cache.store.CacheStore;
-import org.apache.ignite.cluster.BaselineNode;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.lang.IgnitePredicate;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.testframework.GridTestUtils;
-
-/**
- * Tests the recovery after a dynamic cache start failure.
- */
-public abstract class IgniteAbstractDynamicCacheStartFailTest extends GridCacheAbstractSelfTest {
-    /** */
-    private static final String DYNAMIC_CACHE_NAME = "TestDynamicCache";
-
-    /** */
-    private static final String CLIENT_GRID_NAME = "client";
-
-    /** */
-    protected static final String EXISTING_CACHE_NAME = "existing-cache";;
-
-    /** */
-    private static final int PARTITION_COUNT = 16;
-
-    /** Coordinator node index. */
-    private int crdIdx = 0;
-
-    /** {@inheritDoc} */
-    @Override protected int gridCount() {
-        return 3;
-    }
-
-    /**
-     * Returns {@code true} if persistence is enabled.
-     *
-     * @return {@code true} if persistence is enabled.
-     */
-    protected boolean persistenceEnabled() {
-        return false;
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testBrokenAffinityFunStartOnServerFailedOnClient() throws Exception {
-        final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnServerFailedOnClient";
-
-        IgniteConfiguration clientCfg = getConfiguration(clientName);
-
-        clientCfg.setClientMode(true);
-
-        Ignite client = startGrid(clientName, clientCfg);
-
-        CacheConfiguration cfg = new CacheConfiguration();
-
-        cfg.setName(DYNAMIC_CACHE_NAME + "-server-1");
-
-        cfg.setAffinity(new BrokenAffinityFunction(false, clientName));
-
-        try {
-            IgniteCache cache = ignite(0).getOrCreateCache(cfg);
-        }
-        catch (CacheException e) {
-            fail("Exception should not be thrown.");
-        }
-
-        stopGrid(clientName);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testBrokenAffinityFunStartOnServerFailedOnServer() throws Exception {
-        final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnServerFailedOnServer";
-
-        IgniteConfiguration clientCfg = getConfiguration(clientName);
-
-        clientCfg.setClientMode(true);
-
-        Ignite client = startGrid(clientName, clientCfg);
-
-        CacheConfiguration cfg = new CacheConfiguration();
-
-        cfg.setName(DYNAMIC_CACHE_NAME + "-server-2");
-
-        cfg.setAffinity(new BrokenAffinityFunction(false, getTestIgniteInstanceName(0)));
-
-        try {
-            IgniteCache cache = ignite(0).getOrCreateCache(cfg);
-
-            fail("Expected exception was not thrown.");
-        }
-        catch (CacheException e) {
-        }
-
-        stopGrid(clientName);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testBrokenAffinityFunStartOnClientFailOnServer() throws Exception {
-        final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnClientFailOnServer";
-
-        IgniteConfiguration clientCfg = getConfiguration(clientName);
-
-        clientCfg.setClientMode(true);
-
-        Ignite client = startGrid(clientName, clientCfg);
-
-        CacheConfiguration cfg = new CacheConfiguration();
-
-        cfg.setName(DYNAMIC_CACHE_NAME + "-client-2");
-
-        cfg.setAffinity(new BrokenAffinityFunction(false, getTestIgniteInstanceName(0)));
-
-        try {
-            IgniteCache cache = client.getOrCreateCache(cfg);
-
-            fail("Expected exception was not thrown.");
-        }
-        catch (CacheException e) {
-        }
-
-        stopGrid(clientName);
-    }
-
-    /**
-     * Test cache start with broken affinity function that throws an exception on all nodes.
-     */
-    public void testBrokenAffinityFunOnAllNodes() {
-        final boolean failOnAllNodes = true;
-        final int unluckyNode = 0;
-        final int unluckyCfg = 1;
-        final int numOfCaches = 3;
-        final int initiator = 0;
-
-        testDynamicCacheStart(
-            createCacheConfigsWithBrokenAffinityFun(
-                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
-            initiator);
-    }
-
-    /**
-     * Test cache start with broken affinity function that throws an exception on initiator node.
-     */
-    public void testBrokenAffinityFunOnInitiator() {
-        final boolean failOnAllNodes = false;
-        final int unluckyNode = 1;
-        final int unluckyCfg = 1;
-        final int numOfCaches = 3;
-        final int initiator = 1;
-
-        testDynamicCacheStart(
-            createCacheConfigsWithBrokenAffinityFun(
-                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
-            initiator);
-    }
-
-    /**
-     * Test cache start with broken affinity function that throws an exception on non-initiator node.
-     */
-    public void testBrokenAffinityFunOnNonInitiator() {
-        final boolean failOnAllNodes = false;
-        final int unluckyNode = 1;
-        final int unluckyCfg = 1;
-        final int numOfCaches = 3;
-        final int initiator = 2;
-
-        testDynamicCacheStart(
-            createCacheConfigsWithBrokenAffinityFun(
-                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
-            initiator);
-    }
-
-    /**
-     * Test cache start with broken affinity function that throws an exception on coordinator node.
-     */
-    public void testBrokenAffinityFunOnCoordinatorDiffInitiator() {
-        final boolean failOnAllNodes = false;
-        final int unluckyNode = crdIdx;
-        final int unluckyCfg = 1;
-        final int numOfCaches = 3;
-        final int initiator = (crdIdx + 1) % gridCount();
-
-        testDynamicCacheStart(
-            createCacheConfigsWithBrokenAffinityFun(
-                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
-            initiator);
-    }
-
-    /**
-     * Test cache start with broken affinity function that throws an exception on initiator node.
-     */
-    public void testBrokenAffinityFunOnCoordinator() {
-        final boolean failOnAllNodes = false;
-        final int unluckyNode = crdIdx;
-        final int unluckyCfg = 1;
-        final int numOfCaches = 3;
-        final int initiator = crdIdx;
-
-        testDynamicCacheStart(
-            createCacheConfigsWithBrokenAffinityFun(
-                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
-            initiator);
-    }
-
-    /**
-     * Tests cache start with node filter and broken affinity function that throws an exception on initiator node.
-     */
-    public void testBrokenAffinityFunWithNodeFilter() {
-        final boolean failOnAllNodes = false;
-        final int unluckyNode = 0;
-        final int unluckyCfg = 0;
-        final int numOfCaches = 1;
-        final int initiator = 0;
-
-        testDynamicCacheStart(
-            createCacheConfigsWithBrokenAffinityFun(
-                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, true),
-            initiator);
-    }
-
-    /**
-     * Tests cache start with broken cache store that throws an exception on all nodes.
-     */
-    public void testBrokenCacheStoreOnAllNodes() {
-        final boolean failOnAllNodes = true;
-        final int unluckyNode = 0;
-        final int unluckyCfg = 1;
-        final int numOfCaches = 3;
-        final int initiator = 0;
-
-        testDynamicCacheStart(
-            createCacheConfigsWithBrokenCacheStore(
-                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
-            initiator);
-    }
-
-    /**
-     * Tests cache start with broken cache store that throws an exception on initiator node.
-     */
-    public void testBrokenCacheStoreOnInitiator() {
-        final boolean failOnAllNodes = false;
-        final int unluckyNode = 1;
-        final int unluckyCfg = 1;
-        final int numOfCaches = 3;
-        final int initiator = 1;
-
-        testDynamicCacheStart(
-            createCacheConfigsWithBrokenCacheStore(
-                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
-            initiator);
-    }
-
-    /**
-     * Tests cache start with broken cache store that throws an exception on non-initiator node.
-     */
-    public void testBrokenCacheStoreOnNonInitiator() {
-        final boolean failOnAllNodes = false;
-        final int unluckyNode = 1;
-        final int unluckyCfg = 1;
-        final int numOfCaches = 3;
-        final int initiator = 2;
-
-        testDynamicCacheStart(
-            createCacheConfigsWithBrokenCacheStore(
-                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
-            initiator);
-    }
-
-    /**
-     *  Tests cache start with broken cache store that throws an exception on initiator node.
-     */
-    public void testBrokenCacheStoreOnCoordinatorDiffInitiator() {
-        final boolean failOnAllNodes = false;
-        final int unluckyNode = crdIdx;
-        final int unluckyCfg = 1;
-        final int numOfCaches = 3;
-        final int initiator = (crdIdx + 1) % gridCount();
-
-        testDynamicCacheStart(
-            createCacheConfigsWithBrokenCacheStore(
-                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
-            initiator);
-    }
-
-    /**
-     *  Tests cache start with broken cache store that throws an exception on coordinator node.
-     */
-    public void testBrokenCacheStoreFunOnCoordinator() {
-        final boolean failOnAllNodes = false;
-        final int unluckyNode = crdIdx;
-        final int unluckyCfg = 1;
-        final int numOfCaches = 3;
-        final int initiator = crdIdx;
-
-        testDynamicCacheStart(
-            createCacheConfigsWithBrokenCacheStore(
-                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
-            initiator);
-    }
-
-    /**
-     *  Tests multiple creation of cache with broken affinity function.
-     */
-    public void testCreateCacheMultipleTimes() {
-        final boolean failOnAllNodes = false;
-        final int unluckyNode = 1;
-        final int unluckyCfg = 0;
-        final int numOfAttempts = 15;
-
-        CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun(
-            failOnAllNodes, unluckyNode, unluckyCfg, 1, false).get(0);
-
-        for (int i = 0; i < numOfAttempts; ++i) {
-            try {
-                IgniteCache cache = ignite(0).getOrCreateCache(cfg);
-
-                fail("Expected exception was not thrown");
-            }
-            catch (CacheException e) {
-            }
-        }
-    }
-
-    /**
-     * Tests that a cache with the same name can be started after failure if cache configuration is corrected.
-     *
-     * @throws Exception If test failed.
-     */
-    public void testCacheStartAfterFailure() throws Exception {
-        CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun(
-            false, 1, 0, 1, false).get(0);
-
-        GridTestUtils.assertThrows(log, new Callable<Object>() {
-            @Override public Object call() throws Exception {
-                grid(0).getOrCreateCache(cfg);
-                return null;
-            }
-        }, CacheException.class, null);
-
-        // Correct the cache configuration. Default constructor creates a good affinity function.
-        cfg.setAffinity(new BrokenAffinityFunction());
-
-        IgniteCache<Integer, Value> cache = grid(0).getOrCreateCache(createCacheConfiguration(EXISTING_CACHE_NAME));
-
-        checkCacheOperations(cache);
-    }
-
-    /**
-     * Tests that other cache (existed before the failed start) is still operable after the failure.
-     *
-     * @throws Exception If test failed.
-     */
-    public void testExistingCacheAfterFailure() throws Exception {
-        IgniteCache<Integer, Value> cache = grid(0).getOrCreateCache(createCacheConfiguration(EXISTING_CACHE_NAME));
-
-        CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun(
-            false, 1, 0, 1, false).get(0);
-
-        GridTestUtils.assertThrows(log, new Callable<Object>() {
-            @Override public Object call() throws Exception {
-                grid(0).getOrCreateCache(cfg);
-                return null;
-            }
-        }, CacheException.class, null);
-
-        checkCacheOperations(cache);
-    }
-
-    /**
-     * Tests that other cache works as expected after the failure and further topology changes.
-     *
-     * @throws Exception If test failed.
-     */
-    public void testTopologyChangesAfterFailure() throws Exception {
-        final String clientName = "testTopologyChangesAfterFailure";
-
-        IgniteCache<Integer, Value> cache = grid(0).getOrCreateCache(createCacheConfiguration(EXISTING_CACHE_NAME));
-
-        checkCacheOperations(cache);
-
-        CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun(
-            false, 0, 0, 1, false).get(0);
-
-        GridTestUtils.assertThrows(log, new Callable<Object>() {
-            @Override public Object call() throws Exception {
-                grid(0).getOrCreateCache(cfg);
-                return null;
-            }
-        }, CacheException.class, null);
-
-        awaitPartitionMapExchange();
-
-        checkCacheOperations(cache);
-
-        // Start a new server node and check cache operations.
-        Ignite serverNode = startGrid(gridCount() + 1);
-
-        if (persistenceEnabled()) {
-            // Start a new client node to perform baseline change.
-            // TODO: This change is workaround:
-            // Sometimes problem with caches configuration deserialization from test thread arises.
-            final String clientName1 = "baseline-changer";
-
-            IgniteConfiguration clientCfg = getConfiguration(clientName1);
-
-            clientCfg.setClientMode(true);
-
-            Ignite clientNode = startGrid(clientName1, clientCfg);
-
-            List<BaselineNode> baseline = new ArrayList<>(grid(0).cluster().currentBaselineTopology());
-
-            baseline.add(serverNode.cluster().localNode());
-
-            clientNode.cluster().setBaselineTopology(baseline);
-        }
-
-        awaitPartitionMapExchange();
-
-        checkCacheOperations(serverNode.cache(EXISTING_CACHE_NAME));
-
-        // Start a new client node and check cache operations.
-        IgniteConfiguration clientCfg = getConfiguration(clientName);
-
-        clientCfg.setClientMode(true);
-
-        Ignite clientNode = startGrid(clientName, clientCfg);
-
-        checkCacheOperations(clientNode.cache(EXISTING_CACHE_NAME));
-    }
-
-    public void testConcurrentClientNodeJoins() throws Exception {
-        final int clientCnt = 3;
-        final int numberOfAttempts = 5;
-
-        IgniteCache<Integer, Value> cache = grid(0).getOrCreateCache(createCacheConfiguration(EXISTING_CACHE_NAME));
-
-        final AtomicInteger attemptCnt = new AtomicInteger();
-        final CountDownLatch stopLatch = new CountDownLatch(clientCnt);
-
-        IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Object>() {
-            @Override public Object call() throws Exception {
-                String clientName = Thread.currentThread().getName();
-
-                try {
-                    for (int i = 0; i < numberOfAttempts; ++i) {
-                        int uniqueCnt = attemptCnt.getAndIncrement();
-
-                        IgniteConfiguration clientCfg = getConfiguration(clientName + uniqueCnt);
-
-                        clientCfg.setClientMode(true);
-
-                        final Ignite clientNode = startGrid(clientName, clientCfg);
-
-                        CacheConfiguration cfg = new CacheConfiguration();
-
-                        cfg.setName(clientName + uniqueCnt);
-
-                        String instanceName = getTestIgniteInstanceName(uniqueCnt % gridCount());
-
-                        cfg.setAffinity(new BrokenAffinityFunction(false, instanceName));
-
-                        GridTestUtils.assertThrows(log, new Callable<Object>() {
-                            @Override public Object call() throws Exception {
-                                clientNode.getOrCreateCache(cfg);
-                                return null;
-                            }
-                        }, CacheException.class, null);
-
-                        stopGrid(clientName, true);
-                    }
-                }
-                catch (Exception e) {
-                    fail("Unexpected exception: " + e.getMessage());
-                }
-                finally {
-                    stopLatch.countDown();
-                }
-
-                return null;
-            }
-        }, clientCnt, "start-client-thread");
-
-        stopLatch.await();
-
-        assertEquals(numberOfAttempts * clientCnt, attemptCnt.get());
-
-        checkCacheOperations(cache);
-    }
-
-    protected void testDynamicCacheStart(final Collection<CacheConfiguration> cfgs, final int initiatorId) {
-        assert initiatorId < gridCount();
-
-        GridTestUtils.assertThrows(log, new Callable<Object>() {
-            @Override public Object call() throws Exception {
-                grid(initiatorId).getOrCreateCaches(cfgs);
-                return null;
-            }
-        }, CacheException.class, null);
-
-        for (CacheConfiguration cfg: cfgs) {
-            IgniteCache cache = grid(initiatorId).cache(cfg.getName());
-
-            assertNull(cache);
-        }
-    }
-
-    /**
-     * Creates new cache configuration with the given name.
-     *
-     * @param cacheName Cache name.
-     * @return New cache configuration.
-     */
-    protected CacheConfiguration createCacheConfiguration(String cacheName) {
-        CacheConfiguration cfg = new CacheConfiguration()
-            .setName(cacheName)
-            .setCacheMode(CacheMode.PARTITIONED)
-            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
-            .setAffinity(new BrokenAffinityFunction());
-
-        return cfg;
-    }
-
-    /**
-     * Create list of cache configurations.
-     *
-     * @param failOnAllNodes {@code true} if affinity function should be broken on all nodes.
-     * @param unluckyNode Node, where exception is raised.
-     * @param unluckyCfg Unlucky cache configuration number.
-     * @param cacheNum Number of caches.
-     * @param useFilter {@code true} if NodeFilter should be used.
-     *
-     * @return List of cache configurations.
-     */
-    protected List<CacheConfiguration> createCacheConfigsWithBrokenAffinityFun(
-        boolean failOnAllNodes,
-        int unluckyNode,
-        final int unluckyCfg,
-        int cacheNum,
-        boolean useFilter
-    ) {
-        assert unluckyCfg >= 0 && unluckyCfg < cacheNum;
-
-        final UUID uuid = ignite(unluckyNode).cluster().localNode().id();
-
-        List<CacheConfiguration> cfgs = new ArrayList<>();
-
-        for (int i = 0; i < cacheNum; ++i) {
-            CacheConfiguration cfg = createCacheConfiguration(DYNAMIC_CACHE_NAME + "-" + i);
-
-            if (i == unluckyCfg)
-                cfg.setAffinity(new BrokenAffinityFunction(failOnAllNodes, getTestIgniteInstanceName(unluckyNode)));
-
-            if (useFilter)
-                cfg.setNodeFilter(new NodeFilter(uuid));
-
-            cfgs.add(cfg);
-        }
-
-        return cfgs;
-    }
-
-    /**
-     * Create list of cache configurations.
-     *
-     * @param failOnAllNodes {@code true} if cache store should be broken on all nodes.
-     * @param unluckyNode Node, where exception is raised.
-     * @param unluckyCfg Unlucky cache configuration number.
-     * @param cacheNum Number of caches.
-     * @param useFilter {@code true} if NodeFilter should be used.
-     *
-     * @return List of cache configurations.
-     */
-    protected List<CacheConfiguration> createCacheConfigsWithBrokenCacheStore(
-        boolean failOnAllNodes,
-        int unluckyNode,
-        int unluckyCfg,
-        int cacheNum,
-        boolean useFilter
-    ) {
-        assert unluckyCfg >= 0 && unluckyCfg < cacheNum;
-
-        final UUID uuid = ignite(unluckyNode).cluster().localNode().id();
-
-        List<CacheConfiguration> cfgs = new ArrayList<>();
-
-        for (int i = 0; i < cacheNum; ++i) {
-            CacheConfiguration cfg = new CacheConfiguration();
-
-            cfg.setName(DYNAMIC_CACHE_NAME + "-" + i);
-
-            if (i == unluckyCfg)
-                cfg.setCacheStoreFactory(new BrokenStoreFactory(failOnAllNodes, getTestIgniteInstanceName(unluckyNode)));
-
-            if (useFilter)
-                cfg.setNodeFilter(new NodeFilter(uuid));
-
-            cfgs.add(cfg);
-        }
-
-        return cfgs;
-    }
-
-    /**
-     * Test the basic cache operations.
-     *
-     * @param cache Cache.
-     * @throws Exception If test failed.
-     */
-    protected void checkCacheOperations(IgniteCache<Integer, Value> cache) throws Exception {
-        int cnt = 1000;
-
-        // Check cache operations.
-        for (int i = 0; i < cnt; ++i)
-            cache.put(i, new Value(i));
-
-        for (int i = 0; i < cnt; ++i) {
-            Value v = cache.get(i);
-
-            assertNotNull(v);
-            assertEquals(i, v.getValue());
-        }
-
-        // Check Data Streamer functionality.
-        try (IgniteDataStreamer<Integer, Value> streamer = grid(0).dataStreamer(cache.getName())) {
-            for (int i = 0; i < 10_000; ++i)
-                streamer.addData(i, new Value(i));
-        }
-    }
-
-    /**
-     *
-     */
-    public static class Value {
-        @QuerySqlField
-        private final int fieldVal;
-
-        public Value(int fieldVal) {
-            this.fieldVal = fieldVal;
-        }
-
-        public int getValue() {
-            return fieldVal;
-        }
-    }
-
-    /**
-     * Filter specifying on which node the cache should be started.
-     */
-    public static class NodeFilter implements IgnitePredicate<ClusterNode> {
-        /** Cache should be created node with certain UUID. */
-        public UUID uuid;
-
-        /**
-         * @param uuid node ID.
-         */
-        public NodeFilter(UUID uuid) {
-            this.uuid = uuid;
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean apply(ClusterNode clusterNode) {
-            return clusterNode.id().equals(uuid);
-        }
-    }
-
-    /**
-     * Affinity function that throws an exception when affinity nodes are calculated on the given node.
-     */
-    public static class BrokenAffinityFunction extends RendezvousAffinityFunction {
-        /** */
-        private static final long serialVersionUID = 0L;
-
-        /** */
-        @IgniteInstanceResource
-        private Ignite ignite;
-
-        /** Exception should arise on all nodes. */
-        private boolean eOnAllNodes = false;
-
-        /** Exception should arise on node with certain name. */
-        private String gridName;
-
-        /**
-         * Constructs a good affinity function.
-         */
-        public BrokenAffinityFunction() {
-            super(false, PARTITION_COUNT);
-            // No-op.
-        }
-
-        /**
-         * @param eOnAllNodes {@code True} if exception should be thrown on all nodes.
-         * @param gridName Exception should arise on node with certain name.
-         */
-        public BrokenAffinityFunction(boolean eOnAllNodes, String gridName) {
-            super(false, PARTITION_COUNT);
-
-            this.eOnAllNodes = eOnAllNodes;
-            this.gridName = gridName;
-        }
-
-        /** {@inheritDoc} */
-        @Override public List<List<ClusterNode>> assignPartitions(AffinityFunctionContext affCtx) {
-            if (eOnAllNodes || ignite.name().equals(gridName))
-                throw new IllegalStateException("Simulated exception [locNodeId="
-                    + ignite.cluster().localNode().id() + "]");
-            else
-                return super.assignPartitions(affCtx);
-        }
-    }
-
-    /**
-     * Factory that throws an exception is got created.
-     */
-    public static class BrokenStoreFactory implements Factory<CacheStore<Integer, String>> {
-        /** */
-        @IgniteInstanceResource
-        private Ignite ignite;
-
-        /** Exception should arise on all nodes. */
-        boolean eOnAllNodes = true;
-
-        /** Exception should arise on node with certain name. */
-        public static String gridName;
-
-        /**
-         * @param eOnAllNodes {@code True} if exception should be thrown on all nodes.
-         * @param gridName Exception should arise on node with certain name.
-         */
-        public BrokenStoreFactory(boolean eOnAllNodes, String gridName) {
-            this.eOnAllNodes = eOnAllNodes;
-
-            this.gridName = gridName;
-        }
-
-        /** {@inheritDoc} */
-        @Override public CacheStore<Integer, String> create() {
-            if (eOnAllNodes || ignite.name().equals(gridName))
-                throw new IllegalStateException("Simulated exception [locNodeId="
-                    + ignite.cluster().localNode().id() + "]");
-            else
-                return null;
-        }
-    }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.io.ObjectInputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.CacheException;
+import javax.cache.configuration.Factory;
+import javax.management.Attribute;
+import javax.management.AttributeList;
+import javax.management.AttributeNotFoundException;
+import javax.management.InstanceAlreadyExistsException;
+import javax.management.InstanceNotFoundException;
+import javax.management.IntrospectionException;
+import javax.management.InvalidAttributeValueException;
+import javax.management.ListenerNotFoundException;
+import javax.management.MBeanException;
+import javax.management.MBeanInfo;
+import javax.management.MBeanRegistrationException;
+import javax.management.MBeanServer;
+import javax.management.NotCompliantMBeanException;
+import javax.management.NotificationFilter;
+import javax.management.NotificationListener;
+import javax.management.ObjectInstance;
+import javax.management.ObjectName;
+import javax.management.OperationsException;
+import javax.management.QueryExp;
+import javax.management.ReflectionException;
+import javax.management.loading.ClassLoaderRepository;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.affinity.AffinityFunctionContext;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.cache.store.CacheStore;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Tests the recovery after a dynamic cache start failure.
+ */
+public abstract class IgniteAbstractDynamicCacheStartFailTest extends GridCacheAbstractSelfTest {
+    /** */
+    private static final String DYNAMIC_CACHE_NAME = "TestDynamicCache";
+
+    /** */
+    private static final String CLIENT_GRID_NAME = "client";
+
+    /** */
+    protected static final String EXISTING_CACHE_NAME = "existing-cache";;
+
+    /** */
+    private static final int PARTITION_COUNT = 16;
+
+    /** Failure MBean server. */
+    private static FailureMBeanServer mbSrv;
+
+    /** Coordinator node index. */
+    private int crdIdx = 0;
+
+    /** {@inheritDoc} */
+    @Override protected int gridCount() {
+        return 3;
+    }
+
+    /**
+     * Returns {@code true} if persistence is enabled.
+     *
+     * @return {@code true} if persistence is enabled.
+     */
+    protected boolean persistenceEnabled() {
+        return false;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBrokenAffinityFunStartOnServerFailedOnClient() throws Exception {
+        final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnServerFailedOnClient";
+
+        IgniteConfiguration clientCfg = getConfiguration(clientName);
+
+        clientCfg.setClientMode(true);
+
+        Ignite client = startGrid(clientName, clientCfg);
+
+        CacheConfiguration cfg = new CacheConfiguration();
+
+        cfg.setName(DYNAMIC_CACHE_NAME + "-server-1");
+
+        cfg.setAffinity(new BrokenAffinityFunction(false, clientName));
+
+        try {
+            IgniteCache cache = ignite(0).getOrCreateCache(cfg);
+        }
+        catch (CacheException e) {
+            fail("Exception should not be thrown.");
+        }
+
+        stopGrid(clientName);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBrokenAffinityFunStartOnServerFailedOnServer() throws Exception {
+        final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnServerFailedOnServer";
+
+        IgniteConfiguration clientCfg = getConfiguration(clientName);
+
+        clientCfg.setClientMode(true);
+
+        Ignite client = startGrid(clientName, clientCfg);
+
+        CacheConfiguration cfg = new CacheConfiguration();
+
+        cfg.setName(DYNAMIC_CACHE_NAME + "-server-2");
+
+        cfg.setAffinity(new BrokenAffinityFunction(false, getTestIgniteInstanceName(0)));
+
+        try {
+            IgniteCache cache = ignite(0).getOrCreateCache(cfg);
+
+            fail("Expected exception was not thrown.");
+        }
+        catch (CacheException e) {
+        }
+
+        stopGrid(clientName);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBrokenAffinityFunStartOnClientFailOnServer() throws Exception {
+        final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnClientFailOnServer";
+
+        IgniteConfiguration clientCfg = getConfiguration(clientName);
+
+        clientCfg.setClientMode(true);
+
+        Ignite client = startGrid(clientName, clientCfg);
+
+        CacheConfiguration cfg = new CacheConfiguration();
+
+        cfg.setName(DYNAMIC_CACHE_NAME + "-client-2");
+
+        cfg.setAffinity(new BrokenAffinityFunction(false, getTestIgniteInstanceName(0)));
+
+        try {
+            IgniteCache cache = client.getOrCreateCache(cfg);
+
+            fail("Expected exception was not thrown.");
+        }
+        catch (CacheException e) {
+        }
+
+        stopGrid(clientName);
+    }
+
+    /**
+     * Test cache start with broken affinity function that throws an exception on all nodes.
+     */
+    public void testBrokenAffinityFunOnAllNodes() {
+        final boolean failOnAllNodes = true;
+        final int unluckyNode = 0;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = 0;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     * Test cache start with broken affinity function that throws an exception on initiator node.
+     */
+    public void testBrokenAffinityFunOnInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = 1;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     * Test cache start with broken affinity function that throws an exception on non-initiator node.
+     */
+    public void testBrokenAffinityFunOnNonInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = 2;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     * Test cache start with broken affinity function that throws an exception on coordinator node.
+     */
+    public void testBrokenAffinityFunOnCoordinatorDiffInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = crdIdx;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = (crdIdx + 1) % gridCount();
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     * Test cache start with broken affinity function that throws an exception on initiator node.
+     */
+    public void testBrokenAffinityFunOnCoordinator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = crdIdx;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = crdIdx;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     * Tests cache start with node filter and broken affinity function that throws an exception on initiator node.
+     */
+    public void testBrokenAffinityFunWithNodeFilter() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 0;
+        final int unluckyCfg = 0;
+        final int numOfCaches = 1;
+        final int initiator = 0;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, true),
+            initiator);
+    }
+
+    /**
+     * Tests cache start with broken cache store that throws an exception on all nodes.
+     */
+    public void testBrokenCacheStoreOnAllNodes() {
+        final boolean failOnAllNodes = true;
+        final int unluckyNode = 0;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = 0;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     * Tests cache start with broken cache store that throws an exception on initiator node.
+     */
+    public void testBrokenCacheStoreOnInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = 1;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     * Tests cache start that throws an Ignite checked exception on initiator node.
+     */
+    public void testThrowsIgniteCheckedExceptionOnInitiator() {
+        final int unluckyNode = 1;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = 1;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithFailureMbServer(unluckyNode, unluckyCfg, numOfCaches),
+            initiator);
+    }
+
+    /**
+     * Tests cache start with broken cache store that throws an exception on non-initiator node.
+     */
+    public void testBrokenCacheStoreOnNonInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = 2;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     * Tests cache start that throws an Ignite checked exception on non-initiator node.
+     */
+    public void testThrowsIgniteCheckedExceptionOnNonInitiator() {
+        final int unluckyNode = 1;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = 2;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithFailureMbServer(unluckyNode, unluckyCfg, numOfCaches),
+            initiator);
+    }
+
+    /**
+     *  Tests cache start with broken cache store that throws an exception on initiator node.
+     */
+    public void testBrokenCacheStoreOnCoordinatorDiffInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = crdIdx;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = (crdIdx + 1) % gridCount();
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     *  Tests cache start that throws an Ignite checked exception on coordinator node
+     *  that doesn't initiator node.
+     */
+    public void testThrowsIgniteCheckedExceptionOnCoordinatorDiffInitiator() {
+        final int unluckyNode = crdIdx;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = (crdIdx + 1) % gridCount();
+
+        testDynamicCacheStart(
+            createCacheConfigsWithFailureMbServer(unluckyNode, unluckyCfg, numOfCaches),
+            initiator);
+    }
+
+    /**
+     *  Tests cache start with broken cache store that throws an exception on coordinator node.
+     */
+    public void testBrokenCacheStoreFunOnCoordinator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = crdIdx;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = crdIdx;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     *  Tests cache start that throws an Ignite checked exception on coordinator node.
+     */
+    public void testThrowsIgniteCheckedExceptionOnCoordinator() {
+        final int unluckyNode = crdIdx;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = crdIdx;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithFailureMbServer(unluckyNode, unluckyCfg, numOfCaches),
+            initiator);
+    }
+
+    /**
+     *  Tests multiple creation of cache with broken affinity function.
+     */
+    public void testCreateCacheMultipleTimes() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 0;
+        final int numOfAttempts = 15;
+
+        CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun(
+            failOnAllNodes, unluckyNode, unluckyCfg, 1, false).get(0);
+
+        for (int i = 0; i < numOfAttempts; ++i) {
+            try {
+                IgniteCache cache = ignite(0).getOrCreateCache(cfg);
+
+                fail("Expected exception was not thrown");
+            }
+            catch (CacheException e) {
+            }
+        }
+    }
+
+    /**
+     * Tests that a cache with the same name can be started after failure if cache configuration is corrected.
+     *
+     * @throws Exception If test failed.
+     */
+    public void testCacheStartAfterFailure() throws Exception {
+        CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun(
+            false, 1, 0, 1, false).get(0);
+
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                grid(0).getOrCreateCache(cfg);
+                return null;
+            }
+        }, CacheException.class, null);
+
+        // Correct the cache configuration. Default constructor creates a good affinity function.
+        cfg.setAffinity(new BrokenAffinityFunction());
+
+        checkCacheOperations(grid(0).getOrCreateCache(cfg));
+    }
+
+    /**
+     * Tests that other cache (existed before the failed start) is still operable after the failure.
+     *
+     * @throws Exception If test failed.
+     */
+    public void testExistingCacheAfterFailure() throws Exception {
+        IgniteCache<Integer, Value> cache = grid(0).getOrCreateCache(createCacheConfiguration(EXISTING_CACHE_NAME));
+
+        CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun(
+            false, 1, 0, 1, false).get(0);
+
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                grid(0).getOrCreateCache(cfg);
+                return null;
+            }
+        }, CacheException.class, null);
+
+        checkCacheOperations(cache);
+    }
+
+    /**
+     * Tests that other cache works as expected after the failure and further topology changes.
+     *
+     * @throws Exception If test failed.
+     */
+    public void testTopologyChangesAfterFailure() throws Exception {
+        final String clientName = "testTopologyChangesAfterFailure";
+
+        IgniteCache<Integer, Value> cache = grid(0).getOrCreateCache(createCacheConfiguration(EXISTING_CACHE_NAME));
+
+        checkCacheOperations(cache);
+
+        CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun(
+            false, 0, 0, 1, false).get(0);
+
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                grid(0).getOrCreateCache(cfg);
+                return null;
+            }
+        }, CacheException.class, null);
+
+        awaitPartitionMapExchange();
+
+        checkCacheOperations(cache);
+
+        // Start a new server node and check cache operations.
+        Ignite serverNode = startGrid(gridCount() + 1);
+
+        if (persistenceEnabled()) {
+            // Start a new client node to perform baseline change.
+            // TODO: This change is workaround:
+            // Sometimes problem with caches configuration deserialization from test thread arises.
+            final String clientName1 = "baseline-changer";
+
+            IgniteConfiguration clientCfg = getConfiguration(clientName1);
+
+            clientCfg.setClientMode(true);
+
+            Ignite clientNode = startGrid(clientName1, clientCfg);
+
+            List<BaselineNode> baseline = new ArrayList<>(grid(0).cluster().currentBaselineTopology());
+
+            baseline.add(serverNode.cluster().localNode());
+
+            clientNode.cluster().setBaselineTopology(baseline);
+        }
+
+        awaitPartitionMapExchange();
+
+        checkCacheOperations(serverNode.cache(EXISTING_CACHE_NAME));
+
+        // Start a new client node and check cache operations.
+        IgniteConfiguration clientCfg = getConfiguration(clientName);
+
+        clientCfg.setClientMode(true);
+
+        Ignite clientNode = startGrid(clientName, clientCfg);
+
+        checkCacheOperations(clientNode.cache(EXISTING_CACHE_NAME));
+    }
+
+    public void testConcurrentClientNodeJoins() throws Exception {
+        final int clientCnt = 3;
+        final int numberOfAttempts = 5;
+
+        IgniteCache<Integer, Value> cache = grid(0).getOrCreateCache(createCacheConfiguration(EXISTING_CACHE_NAME));
+
+        final AtomicInteger attemptCnt = new AtomicInteger();
+        final CountDownLatch stopLatch = new CountDownLatch(clientCnt);
+
+        IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                String clientName = Thread.currentThread().getName();
+
+                try {
+                    for (int i = 0; i < numberOfAttempts; ++i) {
+                        int uniqueCnt = attemptCnt.getAndIncrement();
+
+                        IgniteConfiguration clientCfg = getConfiguration(clientName + uniqueCnt);
+
+                        clientCfg.setClientMode(true);
+
+                        final Ignite clientNode = startGrid(clientName, clientCfg);
+
+                        CacheConfiguration cfg = new CacheConfiguration();
+
+                        cfg.setName(clientName + uniqueCnt);
+
+                        String instanceName = getTestIgniteInstanceName(uniqueCnt % gridCount());
+
+                        cfg.setAffinity(new BrokenAffinityFunction(false, instanceName));
+
+                        GridTestUtils.assertThrows(log, new Callable<Object>() {
+                            @Override public Object call() throws Exception {
+                                clientNode.getOrCreateCache(cfg);
+                                return null;
+                            }
+                        }, CacheException.class, null);
+
+                        stopGrid(clientName, true);
+                    }
+                }
+                catch (Exception e) {
+                    fail("Unexpected exception: " + e.getMessage());
+                }
+                finally {
+                    stopLatch.countDown();
+                }
+
+                return null;
+            }
+        }, clientCnt, "start-client-thread");
+
+        stopLatch.await();
+
+        assertEquals(numberOfAttempts * clientCnt, attemptCnt.get());
+
+        checkCacheOperations(cache);
+    }
+
+    protected void testDynamicCacheStart(final Collection<CacheConfiguration> cfgs, final int initiatorId) {
+        assert initiatorId < gridCount();
+
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                grid(initiatorId).getOrCreateCaches(cfgs);
+                return null;
+            }
+        }, CacheException.class, null);
+
+        for (CacheConfiguration cfg: cfgs) {
+            IgniteCache cache = grid(initiatorId).cache(cfg.getName());
+
+            assertNull(cache);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration res = super.getConfiguration(igniteInstanceName);
+
+        if (mbSrv == null)
+            mbSrv = new FailureMBeanServer(res.getMBeanServer());
+
+        res.setMBeanServer(mbSrv);
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        mbSrv.clear();
+
+        for (String cacheName : grid(0).cacheNames()) {
+            if (!(EXISTING_CACHE_NAME.equals(cacheName) || DEFAULT_CACHE_NAME.equals(cacheName)))
+                grid(0).cache(cacheName).destroy();
+        }
+    }
+
+    /**
+     * Creates new cache configuration with the given name.
+     *
+     * @param cacheName Cache name.
+     * @return New cache configuration.
+     */
+    protected CacheConfiguration createCacheConfiguration(String cacheName) {
+        CacheConfiguration cfg = new CacheConfiguration()
+            .setName(cacheName)
+            .setCacheMode(CacheMode.PARTITIONED)
+            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+            .setAffinity(new BrokenAffinityFunction());
+
+        return cfg;
+    }
+
+    /**
+     * Create list of cache configurations.
+     *
+     * @param failOnAllNodes {@code true} if affinity function should be broken on all nodes.
+     * @param unluckyNode Node, where exception is raised.
+     * @param unluckyCfg Unlucky cache configuration number.
+     * @param cacheNum Number of caches.
+     * @param useFilter {@code true} if NodeFilter should be used.
+     *
+     * @return List of cache configurations.
+     */
+    protected List<CacheConfiguration> createCacheConfigsWithBrokenAffinityFun(
+        boolean failOnAllNodes,
+        int unluckyNode,
+        final int unluckyCfg,
+        int cacheNum,
+        boolean useFilter
+    ) {
+        assert unluckyCfg >= 0 && unluckyCfg < cacheNum;
+
+        final UUID uuid = ignite(unluckyNode).cluster().localNode().id();
+
+        List<CacheConfiguration> cfgs = new ArrayList<>();
+
+        for (int i = 0; i < cacheNum; ++i) {
+            CacheConfiguration cfg = createCacheConfiguration(DYNAMIC_CACHE_NAME + "-" + i);
+
+            if (i == unluckyCfg)
+                cfg.setAffinity(new BrokenAffinityFunction(failOnAllNodes, getTestIgniteInstanceName(unluckyNode)));
+
+            if (useFilter)
+                cfg.setNodeFilter(new NodeFilter(uuid));
+
+            cfgs.add(cfg);
+        }
+
+        return cfgs;
+    }
+
+    /**
+     * Create list of cache configurations.
+     *
+     * @param failOnAllNodes {@code true} if cache store should be broken on all nodes.
+     * @param unluckyNode Node, where exception is raised.
+     * @param unluckyCfg Unlucky cache configuration number.
+     * @param cacheNum Number of caches.
+     * @param useFilter {@code true} if NodeFilter should be used.
+     *
+     * @return List of cache configurations.
+     */
+    protected List<CacheConfiguration> createCacheConfigsWithBrokenCacheStore(
+        boolean failOnAllNodes,
+        int unluckyNode,
+        int unluckyCfg,
+        int cacheNum,
+        boolean useFilter
+    ) {
+        assert unluckyCfg >= 0 && unluckyCfg < cacheNum;
+
+        final UUID uuid = ignite(unluckyNode).cluster().localNode().id();
+
+        List<CacheConfiguration> cfgs = new ArrayList<>();
+
+        for (int i = 0; i < cacheNum; ++i) {
+            CacheConfiguration cfg = new CacheConfiguration();
+
+            cfg.setName(DYNAMIC_CACHE_NAME + "-" + i);
+
+            if (i == unluckyCfg)
+                cfg.setCacheStoreFactory(new BrokenStoreFactory(failOnAllNodes, getTestIgniteInstanceName(unluckyNode)));
+
+            if (useFilter)
+                cfg.setNodeFilter(new NodeFilter(uuid));
+
+            cfgs.add(cfg);
+        }
+
+        return cfgs;
+    }
+
+    /**
+     * Create list of cache configurations.
+     *
+     * @param unluckyNode Node, where exception is raised.
+     * @param unluckyCfg Unlucky cache configuration number.
+     * @param cacheNum Number of caches.
+     *
+     * @return List of cache configurations.
+     */
+    private List<CacheConfiguration> createCacheConfigsWithFailureMbServer(
+        int unluckyNode,
+        int unluckyCfg,
+        int cacheNum
+    ) {
+        assert unluckyCfg >= 0 && unluckyCfg < cacheNum;
+
+        List<CacheConfiguration> cfgs = new ArrayList<>();
+
+        for (int i = 0; i < cacheNum; ++i) {
+            CacheConfiguration cfg = new CacheConfiguration();
+
+            String cacheName = DYNAMIC_CACHE_NAME + "-" + i;
+
+            cfg.setName(cacheName);
+
+            if (i == unluckyCfg)
+                mbSrv.cache(cacheName);
+
+            cfgs.add(cfg);
+        }
+
+        mbSrv.node(getTestIgniteInstanceName(unluckyNode));
+
+        return cfgs;
+    }
+
+    /**
+     * Test the basic cache operations.
+     *
+     * @param cache Cache.
+     * @throws Exception If test failed.
+     */
+    protected void checkCacheOperations(IgniteCache<Integer, Value> cache) throws Exception {
+        int cnt = 1000;
+
+        // Check cache operations.
+        for (int i = 0; i < cnt; ++i)
+            cache.put(i, new Value(i));
+
+        for (int i = 0; i < cnt; ++i) {
+            Value v = cache.get(i);
+
+            assertNotNull(v);
+            assertEquals(i, v.getValue());
+        }
+
+        // Check Data Streamer functionality.
+        try (IgniteDataStreamer<Integer, Value> streamer = grid(0).dataStreamer(cache.getName())) {
+            for (int i = 0; i < 10_000; ++i)
+                streamer.addData(i, new Value(i));
+        }
+    }
+
+    /**
+     *
+     */
+    public static class Value {
+        @QuerySqlField
+        private final int fieldVal;
+
+        public Value(int fieldVal) {
+            this.fieldVal = fieldVal;
+        }
+
+        public int getValue() {
+            return fieldVal;
+        }
+    }
+
+    /**
+     * Filter specifying on which node the cache should be started.
+     */
+    public static class NodeFilter implements IgnitePredicate<ClusterNode> {
+        /** Cache should be created node with certain UUID. */
+        public UUID uuid;
+
+        /**
+         * @param uuid node ID.
+         */
+        public NodeFilter(UUID uuid) {
+            this.uuid = uuid;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean apply(ClusterNode clusterNode) {
+            return clusterNode.id().equals(uuid);
+        }
+    }
+
+    /**
+     * Affinity function that throws an exception when affinity nodes are calculated on the given node.
+     */
+    public static class BrokenAffinityFunction extends RendezvousAffinityFunction {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** Exception should arise on all nodes. */
+        private boolean eOnAllNodes = false;
+
+        /** Exception should arise on node with certain name. */
+        private String gridName;
+
+        /**
+         * Constructs a good affinity function.
+         */
+        public BrokenAffinityFunction() {
+            super(false, PARTITION_COUNT);
+            // No-op.
+        }
+
+        /**
+         * @param eOnAllNodes {@code True} if exception should be thrown on all nodes.
+         * @param gridName Exception should arise on node with certain name.
+         */
+        public BrokenAffinityFunction(boolean eOnAllNodes, String gridName) {
+            super(false, PARTITION_COUNT);
+
+            this.eOnAllNodes = eOnAllNodes;
+            this.gridName = gridName;
+        }
+
+        /** {@inheritDoc} */
+        @Override public List<List<ClusterNode>> assignPartitions(AffinityFunctionContext affCtx) {
+            if (eOnAllNodes || ignite.name().equals(gridName))
+                throw new IllegalStateException("Simulated exception [locNodeId="
+                    + ignite.cluster().localNode().id() + "]");
+            else
+                return super.assignPartitions(affCtx);
+        }
+    }
+
+    /**
+     * Factory that throws an exception is got created.
+     */
+    public static class BrokenStoreFactory implements Factory<CacheStore<Integer, String>> {
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** Exception should arise on all nodes. */
+        boolean eOnAllNodes = true;
+
+        /** Exception should arise on node with certain name. */
+        public static String gridName;
+
+        /**
+         * @param eOnAllNodes {@code True} if exception should be thrown on all nodes.
+         * @param gridName Exception should arise on node with certain name.
+         */
+        public BrokenStoreFactory(boolean eOnAllNodes, String gridName) {
+            this.eOnAllNodes = eOnAllNodes;
+
+            this.gridName = gridName;
+        }
+
+        /** {@inheritDoc} */
+        @Override public CacheStore<Integer, String> create() {
+            if (eOnAllNodes || ignite.name().equals(gridName))
+                throw new IllegalStateException("Simulated exception [locNodeId="
+                    + ignite.cluster().localNode().id() + "]");
+            else
+                return null;
+        }
+    }
+
+    /** Failure MBean server. */
+    private class FailureMBeanServer implements MBeanServer {
+        /** */
+        private final MBeanServer origin;
+
+        /** Set of caches that must be failure. */
+        private final Set<String> caches = new HashSet<>();
+
+        /** Set of nodes that must be failure. */
+        private final Set<String> nodes = new HashSet<>();
+
+        /** */
+        private FailureMBeanServer(MBeanServer origin) {
+            this.origin = origin;
+        }
+
+        /** Add cache name to failure set. */
+        void cache(String cache) {
+            caches.add('\"' + cache + '\"');
+        }
+
+        /** Add node name to failure set. */
+        void node(String node) {
+            nodes.add(node);
+        }
+
+        /** Clear failure set of caches and set of nodes. */
+        void clear() {
+            caches.clear();
+            nodes.clear();
+        }
+
+        /** {@inheritDoc} */
+        @Override public ObjectInstance registerMBean(Object obj, ObjectName name)
+            throws InstanceAlreadyExistsException, MBeanRegistrationException, NotCompliantMBeanException {
+            String node = name.getKeyProperty("igniteInstanceName");
+
+            if (nodes.contains(node) && caches.contains(name.getKeyProperty("group")))
+                throw new MBeanRegistrationException(new Exception("Simulate exception [node=" + node + ']'));
+
+            return origin.registerMBean(obj, name);
+        }
+
+        /** {@inheritDoc} */
+        @Override public ObjectInstance createMBean(String clsName, ObjectName name)
+            throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException {
+            return origin.createMBean(clsName, name);
+        }
+
+        /** {@inheritDoc} */
+        @Override public ObjectInstance createMBean(String clsName, ObjectName name, ObjectName ldrName)
+            throws ReflectionException, InstanceAlreadyExistsException,
+            MBeanException, NotCompliantMBeanException, InstanceNotFoundException {
+            return origin.createMBean(clsName, name, ldrName);
+        }
+
+        /** {@inheritDoc} */
+        @Override public ObjectInstance createMBean(String clsName, ObjectName name, Object[] params,
+            String[] signature) throws ReflectionException, InstanceAlreadyExistsException,
+            MBeanException, NotCompliantMBeanException {
+            return origin.createMBean(clsName, name, params, signature);
+        }
+
+        /** {@inheritDoc} */
+        @Override public ObjectInstance createMBean(String clsName, ObjectName name, ObjectName ldrName,
+            Object[] params, String[] signature) throws ReflectionException, InstanceAlreadyExistsException,
+            MBeanException, NotCompliantMBeanException, InstanceNotFoundException {
+            return origin.createMBean(clsName, name, ldrName, params, signature);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void unregisterMBean(ObjectName name) throws InstanceNotFoundException, MBeanRegistrationException {
+            origin.unregisterMBean(name);
+        }
+
+        /** {@inheritDoc} */
+        @Override public ObjectInstance getObjectInstance(ObjectName name) throws InstanceNotFoundException {
+            return origin.getObjectInstance(name);
+        }
+
+        /** {@inheritDoc} */
+        @Override public Set<ObjectInstance> queryMBeans(ObjectName name, QueryExp qry) {
+            return origin.queryMBeans(name, qry);
+        }
+
+        /** {@inheritDoc} */
+        @Override public Set<ObjectName> queryNames(ObjectName name, QueryExp qry) {
+            return origin.queryNames(name, qry);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean isRegistered(ObjectName name) {
+            return origin.isRegistered(name);
+        }
+
+        /** {@inheritDoc} */
+        @Override public Integer getMBeanCount() {
+            return origin.getMBeanCount();
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object getAttribute(ObjectName name, String attribute)
+            throws MBeanException, AttributeNotFoundException, InstanceNotFoundException, ReflectionException {
+            return origin.getAttribute(name, attribute);
+        }
+
+        /** {@inheritDoc} */
+        @Override public AttributeList getAttributes(ObjectName name,
+            String[] attrs) throws InstanceNotFoundException, ReflectionException {
+            return origin.getAttributes(name, attrs);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void setAttribute(ObjectName name,
+            Attribute attribute) throws InstanceNotFoundException, AttributeNotFoundException,
+            InvalidAttributeValueException, MBeanException, ReflectionException {
+            origin.setAttribute(name, attribute);
+        }
+
+        /** {@inheritDoc} */
+        @Override public AttributeList setAttributes(ObjectName name,
+            AttributeList attrs) throws InstanceNotFoundException, ReflectionException {
+            return origin.setAttributes(name, attrs);
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object invoke(ObjectName name, String operationName, Object[] params,
+            String[] signature) throws InstanceNotFoundException, MBeanException, ReflectionException {
+            return origin.invoke(name, operationName, params, signature);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String getDefaultDomain() {
+            return origin.getDefaultDomain();
+        }
+
+        /** {@inheritDoc} */
+        @Override public String[] getDomains() {
+            return origin.getDomains();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void addNotificationListener(ObjectName name, NotificationListener lsnr,
+            NotificationFilter filter, Object handback) throws InstanceNotFoundException {
+            origin.addNotificationListener(name, lsnr, filter, handback);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void addNotificationListener(ObjectName name, ObjectName lsnr,
+            NotificationFilter filter, Object handback) throws InstanceNotFoundException {
+            origin.addNotificationListener(name, lsnr, filter, handback);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void removeNotificationListener(ObjectName name,
+            ObjectName lsnr) throws InstanceNotFoundException, ListenerNotFoundException {
+            origin.removeNotificationListener(name, lsnr);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void removeNotificationListener(ObjectName name, ObjectName lsnr,
+            NotificationFilter filter, Object handback) throws InstanceNotFoundException, ListenerNotFoundException {
+            origin.removeNotificationListener(name, lsnr, filter, handback);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void removeNotificationListener(ObjectName name,
+            NotificationListener lsnr) throws InstanceNotFoundException, ListenerNotFoundException {
+            origin.removeNotificationListener(name, lsnr);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void removeNotificationListener(ObjectName name, NotificationListener lsnr,
+            NotificationFilter filter, Object handback) throws InstanceNotFoundException, ListenerNotFoundException {
+            origin.removeNotificationListener(name, lsnr, filter, handback);
+        }
+
+        /** {@inheritDoc} */
+        @Override public MBeanInfo getMBeanInfo(
+            ObjectName name) throws InstanceNotFoundException, IntrospectionException, ReflectionException {
+            return origin.getMBeanInfo(name);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean isInstanceOf(ObjectName name, String clsName) throws InstanceNotFoundException {
+            return origin.isInstanceOf(name, clsName);
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object instantiate(String clsName) throws ReflectionException, MBeanException {
+            return origin.instantiate(clsName);
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object instantiate(String clsName,
+            ObjectName ldrName) throws ReflectionException, MBeanException, InstanceNotFoundException {
+            return origin.instantiate(clsName, ldrName);
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object instantiate(String clsName, Object[] params,
+            String[] signature) throws ReflectionException, MBeanException {
+            return origin.instantiate(clsName, params, signature);
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object instantiate(String clsName, ObjectName ldrName, Object[] params,
+            String[] signature) throws ReflectionException, MBeanException, InstanceNotFoundException {
+            return origin.instantiate(clsName, ldrName, params, signature);
+        }
+
+        /** {@inheritDoc} */
+        @Override @Deprecated public ObjectInputStream deserialize(ObjectName name, byte[] data)
+            throws OperationsException {
+            return origin.deserialize(name, data);
+        }
+
+        /** {@inheritDoc} */
+        @Override @Deprecated public ObjectInputStream deserialize(String clsName, byte[] data)
+            throws OperationsException, ReflectionException {
+            return origin.deserialize(clsName, data);
+        }
+
+        /** {@inheritDoc} */
+        @Override @Deprecated public ObjectInputStream deserialize(String clsName, ObjectName ldrName, byte[] data)
+            throws OperationsException, ReflectionException {
+            return origin.deserialize(clsName, ldrName, data);
+        }
+
+        /** {@inheritDoc} */
+        @Override public ClassLoader getClassLoaderFor(ObjectName mbeanName) throws InstanceNotFoundException {
+            return origin.getClassLoaderFor(mbeanName);
+        }
+
+        /** {@inheritDoc} */
+        @Override public ClassLoader getClassLoader(ObjectName ldrName) throws InstanceNotFoundException {
+            return origin.getClassLoader(ldrName);
+        }
+
+        /** {@inheritDoc} */
+        @Override public ClassLoaderRepository getClassLoaderRepository() {
+            return origin.getClassLoaderRepository();
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/560240f1/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccConfigurationValidationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccConfigurationValidationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccConfigurationValidationTest.java
index c677656..f8a81ed 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccConfigurationValidationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccConfigurationValidationTest.java
@@ -286,6 +286,41 @@ public class CacheMvccConfigurationValidationTest extends GridCommonAbstractTest
     }
 
     /**
+     * Checks if passed in {@code 'Throwable'} has given class in {@code 'cause'} hierarchy
+     * <b>including</b> that throwable itself and it contains passed message.
+     * <p>
+     * Note that this method follows includes {@link Throwable#getSuppressed()}
+     * into check.
+     *
+     * @param t Throwable to check (if {@code null}, {@code false} is returned).
+     * @param cls Cause class to check (if {@code null}, {@code false} is returned).
+     * @param msg Message to check.
+     * @return {@code True} if one of the causing exception is an instance of passed in classes
+     *      and it contains the passed message, {@code false} otherwise.
+     */
+    private boolean hasCauseWithMessage(@Nullable Throwable t, Class<?> cls, String msg) {
+        if (t == null)
+            return false;
+
+        assert cls != null;
+
+        for (Throwable th = t; th != null; th = th.getCause()) {
+            if (cls.isAssignableFrom(th.getClass()) && th.getMessage() != null && th.getMessage().contains(msg))
+                return true;
+
+            for (Throwable n : th.getSuppressed()) {
+                if (hasCauseWithMessage(n, cls, msg))
+                    return true;
+            }
+
+            if (th.getCause() == th)
+                break;
+        }
+
+        return false;
+    }
+
+    /**
      * Make sure cache cannot be started with the given configuration.
      *
      * @param ccfg Cache configuration.
@@ -305,7 +340,7 @@ public class CacheMvccConfigurationValidationTest extends GridCommonAbstractTest
             catch (Exception e) {
                 if (msg != null) {
                     assert e.getMessage() != null : "Error message is null";
-                    assert e.getMessage().contains(msg) : "Wrong error message: " + e.getMessage();
+                    assertTrue(hasCauseWithMessage(e, IgniteCheckedException.class, msg));
                 }
             }
         }

http://git-wip-us.apache.org/repos/asf/ignite/blob/560240f1/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlIllegalSchemaSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlIllegalSchemaSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlIllegalSchemaSelfTest.java
index e56f8a2..a91311a 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlIllegalSchemaSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlIllegalSchemaSelfTest.java
@@ -17,16 +17,19 @@
 
 package org.apache.ignite.internal.processors.query;
 
+import java.util.concurrent.Callable;
+import java.util.function.Consumer;
+import javax.cache.CacheException;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-import javax.cache.CacheException;
-import java.util.concurrent.Callable;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Tests for illegal SQL schemas in node and cache configurations.
@@ -61,17 +64,27 @@ public class SqlIllegalSchemaSelfTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testBadCacheNameDynamic() throws Exception {
-        Ignite node = startGrid();
-
-        GridTestUtils.assertThrows(log, new Callable<Void>() {
-            @Override public Void call() throws Exception {
-                node.getOrCreateCache(new CacheConfiguration().setName(QueryUtils.SCHEMA_SYS));
-
-                return null;
+        doubleConsumerAccept(
+            (node)->{
+                try {
+                    node.getOrCreateCache(new CacheConfiguration().setName(QueryUtils.SCHEMA_SYS));
+                }
+                catch (CacheException e) {
+                    assertTrue(hasCause(e, IgniteCheckedException.class,
+                        "SQL schema name derived from cache name is reserved (please set explicit SQL " +
+                            "schema name through CacheConfiguration.setSqlSchema() or choose another cache name) [" +
+                            "cacheName=IGNITE, schemaName=null]"));
+
+                    return;
+                }
+                catch (Throwable e) {
+                    fail("Exception class is not as expected [expected=" +
+                        CacheException.class + ", actual=" + e.getClass() + ']');
+                }
+
+                fail("Exception has not been thrown.");
             }
-        }, CacheException.class, "SQL schema name derived from cache name is reserved (please set explicit SQL " +
-            "schema name through CacheConfiguration.setSqlSchema() or choose another cache name) [" +
-            "cacheName=IGNITE, schemaName=null]");
+        );
     }
 
     /**
@@ -97,17 +110,27 @@ public class SqlIllegalSchemaSelfTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testBadSchemaLowerDynamic() throws Exception {
-        Ignite node = startGrid();
-
-        GridTestUtils.assertThrows(log, new Callable<Void>() {
-            @Override public Void call() throws Exception {
-                node.getOrCreateCache(
-                    new CacheConfiguration().setName("CACHE").setSqlSchema(QueryUtils.SCHEMA_SYS.toLowerCase())
-                );
-
-                return null;
+        doubleConsumerAccept(
+            (node) -> {
+                try {
+                    node.getOrCreateCache(
+                        new CacheConfiguration().setName("CACHE").setSqlSchema(QueryUtils.SCHEMA_SYS.toLowerCase())
+                    );
+                }
+                catch (CacheException e) {
+                    assertTrue(hasCause(e, IgniteCheckedException.class,
+                        "SQL schema name is reserved (please choose another one) [cacheName=CACHE, schemaName=ignite]"));
+
+                    return;
+                }
+                catch (Throwable e) {
+                    fail("Exception class is not as expected [expected=" +
+                        CacheException.class + ", actual=" + e.getClass() + ']');
+                }
+
+                fail("Exception has not been thrown.");
             }
-        }, CacheException.class, "SQL schema name is reserved (please choose another one) [cacheName=CACHE, schemaName=ignite]");
+        );
     }
 
     /**
@@ -133,18 +156,27 @@ public class SqlIllegalSchemaSelfTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testBadSchemaUpperDynamic() throws Exception {
-        Ignite node = startGrid();
-
-        GridTestUtils.assertThrows(log, new Callable<Void>() {
-            @Override public Void call() throws Exception {
-                node.getOrCreateCache(
-                    new CacheConfiguration().setName("CACHE").setSqlSchema(QueryUtils.SCHEMA_SYS.toUpperCase())
-                );
-
-                return null;
+        doubleConsumerAccept(
+            (node) -> {
+                try {
+                    node.getOrCreateCache(
+                        new CacheConfiguration().setName("CACHE").setSqlSchema(QueryUtils.SCHEMA_SYS.toUpperCase())
+                    );
+                }
+                catch (CacheException e) {
+                    assertTrue(hasCause(e, IgniteCheckedException.class,
+                        "SQL schema name is reserved (please choose another one) [cacheName=CACHE, schemaName=IGNITE]"));
+
+                    return;
+                }
+                catch (Throwable e) {
+                    fail("Exception class is not as expected [expected=" +
+                        CacheException.class + ", actual=" + e.getClass() + ']');
+                }
+
+                fail("Exception has not been thrown.");
             }
-        }, CacheException.class, "SQL schema name is reserved (please choose another one) [cacheName=CACHE, " +
-            "schemaName=IGNITE]");
+        );
     }
 
     /**
@@ -170,18 +202,76 @@ public class SqlIllegalSchemaSelfTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testBadSchemaQuotedDynamic() throws Exception {
+        doubleConsumerAccept(
+            (node) -> {
+                try {
+                    node.getOrCreateCache(
+                        new CacheConfiguration().setName("CACHE")
+                            .setSqlSchema("\"" + QueryUtils.SCHEMA_SYS.toUpperCase() + "\"")
+                    );
+                }
+                catch (CacheException e) {
+                    assertTrue(hasCause(e, IgniteCheckedException.class,
+                        "SQL schema name is reserved (please choose another one) [cacheName=CACHE, schemaName=\"IGNITE\"]"));
+
+                    return;
+                }
+                catch (Throwable e) {
+                    fail("Exception class is not as expected [expected=" +
+                        CacheException.class + ", actual=" + e.getClass() + ']');
+                }
+
+                fail("Exception has not been thrown.");
+            }
+        );
+    }
+
+    /**
+     * Executes double call of consumer's accept method with passed Ignite instance.
+     *
+     * @param cons Consumer.
+     * @throws Exception If failed.
+     */
+    private void doubleConsumerAccept(Consumer<Ignite> cons) throws Exception {
         Ignite node = startGrid();
 
-        GridTestUtils.assertThrows(log, new Callable<Void>() {
-            @Override public Void call() throws Exception {
-                node.getOrCreateCache(
-                    new CacheConfiguration().setName("CACHE")
-                        .setSqlSchema("\"" + QueryUtils.SCHEMA_SYS.toUpperCase() + "\"")
-                );
+        cons.accept(node);
 
-                return null;
+        cons.accept(node);
+    }
+
+    /**
+     * Checks if passed in {@code 'Throwable'} has given class in {@code 'cause'} hierarchy
+     * <b>including</b> that throwable itself and it contains passed message.
+     * <p>
+     * Note that this method follows includes {@link Throwable#getSuppressed()}
+     * into check.
+     *
+     * @param t Throwable to check (if {@code null}, {@code false} is returned).
+     * @param cls Cause class to check (if {@code null}, {@code false} is returned).
+     * @param msg Message to check.
+     * @return {@code True} if one of the causing exception is an instance of passed in classes
+     *      and it contains the passed message, {@code false} otherwise.
+     */
+    private boolean hasCause(@Nullable Throwable t, Class<?> cls, String msg) {
+        if (t == null)
+            return false;
+
+        assert cls != null;
+
+        for (Throwable th = t; th != null; th = th.getCause()) {
+            if (cls.isAssignableFrom(th.getClass()) && F.eq(th.getMessage(), msg))
+                return true;
+
+            for (Throwable n : th.getSuppressed()) {
+                if (hasCause(n, cls, msg))
+                    return true;
             }
-        }, CacheException.class, "SQL schema name is reserved (please choose another one) [cacheName=CACHE, " +
-            "schemaName=\"IGNITE\"]");
+
+            if (th.getCause() == th)
+                break;
+        }
+
+        return false;
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/560240f1/modules/platforms/dotnet/Apache.Ignite.Core.Tests/FailureHandlerTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/FailureHandlerTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/FailureHandlerTest.cs
index 7c447b1..a8aff71 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/FailureHandlerTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/FailureHandlerTest.cs
@@ -71,6 +71,7 @@ namespace Apache.Ignite.Core.Tests
         /// Tests <see cref="StopNodeFailureHandler"/>
         /// </summary>
         [Test]
+        [Ignore("IGNITE-10364")]
         public void TestStopNodeFailureHandler()
         {
            TestFailureHandler(typeof(StopNodeFailureHandler));
@@ -80,6 +81,7 @@ namespace Apache.Ignite.Core.Tests
         /// Tests <see cref="StopNodeOrHaltFailureHandler"/>
         /// </summary>
         [Test]
+        [Ignore("IGNITE-10364")]
         public void TestStopNodeOrHaltFailureHandler()
         {
             TestFailureHandler(typeof(StopNodeOrHaltFailureHandler));