[17/32] ignite git commit: IEP-4 Baseline topology for persistent caches (Phase 1) Contributed by: Dmitriy Govorukhin Dmitry Pavlov Eduard Shangareev Ily

[vo...@apache.org]

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteBaselineAffinityTopologyActivationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteBaselineAffinityTopologyActivationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteBaselineAffinityTopologyActivationTest.java
new file mode 100644
index 0000000..1a9393a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteBaselineAffinityTopologyActivationTest.java
@@ -0,0 +1,993 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.cluster.DetachedClusterNode;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
+import org.apache.ignite.internal.processors.cluster.BaselineTopologyHistory;
+import org.apache.ignite.internal.processors.cluster.BaselineTopologyHistoryItem;
+import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+/**
+ *
+ */
+public class IgniteBaselineAffinityTopologyActivationTest extends GridCommonAbstractTest {
+    /** */
+    private String consId;
+
+    /** Entries count to add to cache. */
+    private static final int ENTRIES_COUNT = 100;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        if (consId != null)
+            cfg.setConsistentId(consId);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration().setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setPersistenceEnabled(true).setMaxSize(10 * 1024 * 1024)
+
+            ).setWalMode(WALMode.LOG_ONLY)
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids(false);
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /**
+     * Verifies that when old but compatible node
+     * (it is the node that once wasn't presented in branchingHistory but hasn't participated in any branching point)
+     * joins the cluster after restart, cluster gets activated.
+     */
+    public void testAutoActivationWithCompatibleOldNode() throws Exception {
+        startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        startGridWithConsistentId("C").active(true);
+
+        stopAllGrids(false);
+
+        startGridWithConsistentId("A");
+        startGridWithConsistentId("B").active(true);
+
+        {
+            IgniteEx nodeA = grid("A");
+
+            assertNotNull(nodeA.cluster().currentBaselineTopology());
+            assertEquals(3, nodeA.cluster().currentBaselineTopology().size());
+
+            assertTrue(nodeA.cluster().active());
+        }
+
+        stopAllGrids(false);
+
+        startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+
+        {
+            IgniteEx nodeA = grid("A");
+
+            assertNotNull(nodeA.cluster().currentBaselineTopology());
+            assertEquals(3, nodeA.cluster().currentBaselineTopology().size());
+
+            assertFalse(nodeA.cluster().active());
+        }
+
+        final Ignite nodeC = startGridWithConsistentId("C");
+
+        boolean active = GridTestUtils.waitForCondition(
+            new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    return nodeC.active();
+                }
+            },
+            10_000
+        );
+
+        assertTrue(active);
+    }
+
+    /**
+     * IgniteCluster::setBaselineTopology(long topVer) should throw an exception
+     * when online node from current BaselineTopology is not presented in topology version.
+     */
+    public void testBltChangeTopVerRemoveOnlineNodeFails() throws Exception {
+        Ignite ignite = startGridWithConsistentId("A");
+
+        ignite.active(true);
+
+        long singleNodeTopVer = ignite.cluster().topologyVersion();
+
+        startGridWithConsistentId("OnlineConsID");
+
+        ignite.cluster().setBaselineTopology(baselineNodes(ignite.cluster().forServers().nodes()));
+
+        boolean expectedExceptionThrown = false;
+
+        try {
+            ignite.cluster().setBaselineTopology(singleNodeTopVer);
+        }
+        catch (IgniteException e) {
+            String errMsg = e.getMessage();
+            assertTrue(errMsg.startsWith("Removing online nodes"));
+            assertTrue(errMsg.contains("[OnlineConsID]"));
+
+            expectedExceptionThrown = true;
+        }
+
+        assertTrue(expectedExceptionThrown);
+    }
+
+    /**
+     * Verifies that online nodes cannot be removed from BaselineTopology (this may change in future).
+     */
+    public void testOnlineNodesCannotBeRemovedFromBaselineTopology() throws Exception {
+        Ignite nodeA = startGridWithConsistentId("A");
+        Ignite nodeB = startGridWithConsistentId("B");
+        Ignite nodeC = startGridWithConsistentId("OnlineConsID");
+
+        nodeC.active(true);
+
+        boolean expectedExceptionIsThrown = false;
+
+        try {
+            nodeC.cluster().setBaselineTopology(Arrays.asList((BaselineNode) nodeA.cluster().localNode(),
+                nodeB.cluster().localNode()));
+        } catch (IgniteException e) {
+            String errMsg = e.getMessage();
+            assertTrue(errMsg.startsWith("Removing online nodes"));
+            assertTrue(errMsg.contains("[OnlineConsID]"));
+
+            expectedExceptionIsThrown = true;
+        }
+
+        assertTrue(expectedExceptionIsThrown);
+    }
+
+    /**
+     *
+     */
+    public void testNodeFailsToJoinWithIncompatiblePreviousBaselineTopology() throws Exception {
+        startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        Ignite nodeC = startGridWithConsistentId("C");
+
+        nodeC.active(true);
+
+        stopAllGrids(false);
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        startGridWithConsistentId("B").active(true);
+
+        nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+        stopAllGrids(false);
+
+        startGridWithConsistentId("C").active(true);
+
+        stopGrid("C", false);
+
+        startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+
+        boolean expectedExceptionThrown = false;
+
+        try {
+            startGridWithConsistentId("C");
+        }
+        catch (IgniteCheckedException e) {
+            expectedExceptionThrown = true;
+
+            if (e.getCause() != null && e.getCause().getCause() != null) {
+                Throwable rootCause = e.getCause().getCause();
+
+                if (!(rootCause instanceof IgniteSpiException) || !rootCause.getMessage().contains("not compatible"))
+                    Assert.fail("Unexpected ignite exception was thrown: " + e);
+            }
+            else
+                throw e;
+        }
+
+        assertTrue("Expected exception wasn't thrown.", expectedExceptionThrown);
+    }
+
+    /**
+     * Verifies scenario when parts of grid were activated independently they are not allowed to join
+     * into the same grid again (due to risks of incompatible data modifications).
+     */
+    public void testIncompatibleBltNodeIsProhibitedToJoinCluster() throws Exception {
+        startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        startGridWithConsistentId("C").active(true);
+
+        stopAllGrids(false);
+
+        startGridWithConsistentId("A");
+        startGridWithConsistentId("B").active(true);
+
+        stopAllGrids(false);
+
+        startGridWithConsistentId("C").active(true);
+
+        stopAllGrids(false);
+
+        startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+
+        boolean expectedExceptionThrown = false;
+
+        try {
+            startGridWithConsistentId("C");
+        }
+        catch (IgniteCheckedException e) {
+            expectedExceptionThrown = true;
+
+            if (e.getCause() != null && e.getCause().getCause() != null) {
+                Throwable rootCause = e.getCause().getCause();
+
+                if (!(rootCause instanceof IgniteSpiException) || !rootCause.getMessage().contains("not compatible"))
+                    Assert.fail("Unexpected ignite exception was thrown: " + e);
+            }
+            else
+                throw e;
+        }
+
+        assertTrue("Expected exception wasn't thrown.", expectedExceptionThrown);
+    }
+
+    /**
+     * Test verifies that node with out-of-data but still compatible Baseline Topology is allowed to join the cluster.
+     */
+    public void testNodeWithOldBltIsAllowedToJoinCluster() throws Exception {
+        final long expectedHash1 = (long)"A".hashCode() + "B".hashCode() + "C".hashCode();
+
+        BaselineTopologyVerifier verifier1 = new BaselineTopologyVerifier() {
+            @Override public void verify(BaselineTopology blt) {
+                assertNotNull(blt);
+
+                assertEquals(3, blt.consistentIds().size());
+
+                long activationHash = U.field(blt, "branchingPntHash");
+
+                assertEquals(expectedHash1, activationHash);
+            }
+        };
+
+        final long expectedHash2 = (long)"A".hashCode() + "B".hashCode();
+
+        BaselineTopologyVerifier verifier2 = new BaselineTopologyVerifier() {
+            @Override public void verify(BaselineTopology blt) {
+                assertNotNull(blt);
+
+                assertEquals(3, blt.consistentIds().size());
+
+                long activationHash = U.field(blt, "branchingPntHash");
+
+                assertEquals(expectedHash2, activationHash);
+            }
+        };
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        Ignite nodeB = startGridWithConsistentId("B");
+        Ignite nodeC = startGridWithConsistentId("C");
+
+        nodeC.active(true);
+        verifyBaselineTopologyOnNodes(verifier1, new Ignite[] {nodeA, nodeB, nodeC});
+
+        stopAllGrids(false);
+
+        nodeA = startGridWithConsistentId("A");
+        nodeB = startGridWithConsistentId("B");
+
+        nodeB.active(true);
+
+        verifyBaselineTopologyOnNodes(verifier2, new Ignite[] {nodeA, nodeB});
+
+        stopAllGrids(false);
+
+        nodeA = startGridWithConsistentId("A");
+        nodeB = startGridWithConsistentId("B");
+        nodeC = startGridWithConsistentId("C");
+
+        verifyBaselineTopologyOnNodes(verifier2, new Ignite[] {nodeA, nodeB, nodeC});
+    }
+
+    /**
+     * Verifies that when new node outside of baseline topology joins active cluster with BLT already set
+     * it receives BLT from the cluster and stores it locally.
+     */
+    public void testNewNodeJoinsToActiveCluster() throws Exception {
+        Ignite nodeA = startGridWithConsistentId("A");
+        Ignite nodeB = startGridWithConsistentId("B");
+        Ignite nodeC = startGridWithConsistentId("C");
+
+        nodeC.active(true);
+
+        BaselineTopologyVerifier verifier1 = new BaselineTopologyVerifier() {
+            @Override public void verify(BaselineTopology blt) {
+                assertNotNull(blt);
+
+                assertEquals(3, blt.consistentIds().size());
+            }
+        };
+
+        verifyBaselineTopologyOnNodes(verifier1, new Ignite[] {nodeA, nodeB, nodeC});
+
+        Ignite nodeD = startGridWithConsistentId("D");
+
+        verifyBaselineTopologyOnNodes(verifier1, new Ignite[] {nodeD});
+
+        stopAllGrids(false);
+
+        nodeD = startGridWithConsistentId("D");
+
+        assertFalse(nodeD.active());
+
+        verifyBaselineTopologyOnNodes(verifier1, new Ignite[] {nodeD});
+    }
+
+    /**
+     *
+     */
+    public void testRemoveNodeFromBaselineTopology() throws Exception {
+        final long expectedActivationHash = (long)"A".hashCode() + "C".hashCode();
+
+        BaselineTopologyVerifier verifier = new BaselineTopologyVerifier() {
+            @Override public void verify(BaselineTopology blt) {
+                assertNotNull(blt);
+
+                assertEquals(2, blt.consistentIds().size());
+
+                long activationHash = U.field(blt, "branchingPntHash");
+
+                assertEquals(expectedActivationHash, activationHash);
+            }
+        };
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        Ignite nodeC = startGridWithConsistentId("C");
+
+        nodeC.active(true);
+
+        stopGrid("B", false);
+
+        nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+        boolean activated = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return grid("A").active();
+            }
+        }, 10_000);
+
+        assertEquals(true, activated);
+
+        verifyBaselineTopologyOnNodes(verifier, new Ignite[] {nodeA, nodeC});
+
+        stopAllGrids(false);
+
+        nodeA = startGridWithConsistentId("A");
+        nodeC = startGridWithConsistentId("C");
+
+        activated = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return grid("A").active();
+            }
+        }, 10_000);
+
+        assertTrue(activated);
+
+        verifyBaselineTopologyOnNodes(verifier, new Ignite[] {nodeA, nodeC});
+    }
+
+    /**
+     *
+     */
+    public void testAddNodeToBaselineTopology() throws Exception {
+        final long expectedActivationHash = (long)"A".hashCode() + "B".hashCode() + "C".hashCode() + "D".hashCode();
+
+        BaselineTopologyVerifier verifier = new BaselineTopologyVerifier() {
+            @Override public void verify(BaselineTopology blt) {
+                assertNotNull(blt);
+
+                assertEquals(4, blt.consistentIds().size());
+
+                long activationHash = U.field(blt, "branchingPntHash");
+
+                assertEquals(expectedActivationHash, activationHash);
+            }
+        };
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        Ignite nodeB = startGridWithConsistentId("B");
+        Ignite nodeC = startGridWithConsistentId("C");
+
+        nodeC.active(true);
+
+        IgniteEx nodeD = (IgniteEx) startGridWithConsistentId("D");
+
+        nodeD.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+        verifyBaselineTopologyOnNodes(verifier, new Ignite[]{nodeA, nodeB, nodeC, nodeD});
+    }
+
+    /**
+     * Verifies that baseline topology is removed successfully through baseline changing API.
+     */
+    public void testRemoveBaselineTopology() throws Exception {
+        BaselineTopologyVerifier verifier = new BaselineTopologyVerifier() {
+            @Override public void verify(BaselineTopology blt) {
+                assertNull(blt);
+            }
+        };
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        Ignite nodeB = startGridWithConsistentId("B");
+        Ignite nodeC = startGridWithConsistentId("C");
+
+        nodeA.active(true);
+
+        nodeA.cluster().setBaselineTopology(null);
+
+        verifyBaselineTopologyOnNodes(verifier, new Ignite[] {nodeA, nodeB, nodeC});
+    }
+
+    /** */
+    private interface BaselineTopologyVerifier {
+        /** */
+        public void verify(BaselineTopology blt);
+    }
+
+    /** */
+    private interface BaselineTopologyHistoryVerifier {
+        /** */
+        public void verify(BaselineTopologyHistory bltHist);
+    }
+
+    /** */
+    private void verifyBaselineTopologyOnNodes(BaselineTopologyVerifier bltVerifier, Ignite[] igs) {
+        for (Ignite ig : igs) {
+            BaselineTopology blt = getBaselineTopology(ig);
+
+            bltVerifier.verify(blt);
+        }
+    }
+
+    /** */
+    private void verifyBaselineTopologyHistoryOnNodes(BaselineTopologyHistoryVerifier bltHistVerifier, Ignite[] igs) {
+        for (Ignite ig : igs) {
+            BaselineTopologyHistory blt = getBaselineTopologyHistory(ig);
+
+            bltHistVerifier.verify(blt);
+        }
+    }
+
+    /** */
+    private Ignite startGridWithConsistentId(String consId) throws Exception {
+        this.consId = consId;
+
+        return startGrid(consId);
+    }
+
+    /**
+     * Verifies that when new node joins already active cluster and new activation request is issued,
+     * no changes to BaselineTopology branching history happen.
+     */
+    public void testActivationHashIsNotUpdatedOnMultipleActivationRequests() throws Exception {
+        final long expectedActivationHash = (long)"A".hashCode();
+
+        BaselineTopologyVerifier verifier = new BaselineTopologyVerifier() {
+            @Override public void verify(BaselineTopology blt) {
+                long activationHash = U.field(blt, "branchingPntHash");
+
+                assertEquals(expectedActivationHash, activationHash);
+            }
+        };
+
+        Ignite nodeA = startGridWithConsistentId("A");
+
+        nodeA.active(true);
+
+        Ignite nodeB = startGridWithConsistentId("B");
+
+        nodeA.active(true);
+
+        verifyBaselineTopologyOnNodes(verifier, new Ignite[] {nodeA, nodeB});
+    }
+
+    /**
+     * Verifies that grid is autoactivated when full BaselineTopology is preset even on one node
+     * and then all other nodes from BaselineTopology are started.
+     */
+    public void testAutoActivationWithBaselineTopologyPreset() throws Exception {
+        Ignite ig = startGridWithConsistentId("A");
+
+        ig.active(true);
+
+        ig.cluster().setBaselineTopology(Arrays.asList(new BaselineNode[] {
+            createBaselineNodeWithConsId("A"), createBaselineNodeWithConsId("B"), createBaselineNodeWithConsId("C")}));
+
+        stopAllGrids();
+
+        final Ignite ig1 = startGridWithConsistentId("A");
+
+        startGridWithConsistentId("B");
+
+        startGridWithConsistentId("C");
+
+        boolean activated = GridTestUtils.waitForCondition(
+            new GridAbsPredicate() {
+               @Override public boolean apply() {
+                   return ig1.active();
+               }
+            },
+            10_000
+        );
+
+        assertTrue(activated);
+    }
+
+    /**
+     * Creates BaselineNode with specific attribute indicating that this node is not client.
+     */
+    private BaselineNode createBaselineNodeWithConsId(String consId) {
+        Map<String, Object> attrs = new HashMap<>();
+
+        attrs.put("org.apache.ignite.cache.client", false);
+
+        return new DetachedClusterNode(consId, attrs);
+    }
+
+    /** */
+    public void testAutoActivationSimple() throws Exception {
+        startGrids(3);
+
+        IgniteEx srv = grid(0);
+
+        srv.active(true);
+
+        createAndFillCache(srv);
+
+        // TODO: final implementation should work with cancel == true.
+        stopAllGrids();
+
+        //note: no call for activation after grid restart
+        startGrids(3);
+
+        final Ignite ig = grid(0);
+
+        boolean clusterActive = GridTestUtils.waitForCondition(
+            new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    return ig.active();
+                }
+            },
+            10_000);
+
+        assertTrue(clusterActive);
+
+        checkDataInCache((IgniteEx) ig);
+    }
+
+    /**
+     *
+     */
+    public void testNoAutoActivationOnJoinNewNodeToInactiveCluster() throws Exception {
+        startGrids(2);
+
+        IgniteEx srv = grid(0);
+
+        srv.active(true);
+
+        awaitPartitionMapExchange();
+
+        assertTrue(srv.active());
+
+        srv.active(false);
+
+        assertFalse(srv.active());
+
+        startGrid(2);
+
+        Thread.sleep(3_000);
+
+        assertFalse(srv.active());
+    }
+
+    /**
+     * Verifies that neither BaselineTopology nor BaselineTopologyHistory are changed when cluster is deactivated.
+     */
+    public void testBaselineTopologyRemainsTheSameOnClusterDeactivation() throws Exception {
+        startGrids(2);
+
+        IgniteEx srv = grid(0);
+
+        srv.active(true);
+
+        awaitPartitionMapExchange();
+
+        assertTrue(srv.active());
+
+        srv.active(false);
+
+        BaselineTopology blt = getBaselineTopology(srv);
+
+        BaselineTopologyHistory bltHist = getBaselineTopologyHistory(srv);
+
+        assertEquals(0, blt.id());
+
+        assertEquals(2, blt.consistentIds().size());
+
+        assertEquals(1, blt.branchingHistory().size());
+
+        assertEquals(0, bltHist.history().size());
+    }
+
+    /**
+     *
+     */
+    public void testBaselineHistorySyncWithNewNode() throws Exception {
+        final long expectedBranchingHash = "A".hashCode() + "B".hashCode() + "C".hashCode();
+
+        BaselineTopologyHistoryVerifier verifier = new BaselineTopologyHistoryVerifier() {
+            @Override public void verify(BaselineTopologyHistory bltHist) {
+                assertNotNull(bltHist);
+
+                assertEquals(1, bltHist.history().size());
+
+                BaselineTopologyHistoryItem histItem = bltHist.history().get(0);
+
+                assertEquals(1, histItem.branchingHistory().size());
+
+                long actualBranchingHash = histItem.branchingHistory().get(0);
+
+                assertEquals(expectedBranchingHash, actualBranchingHash);
+            }
+        };
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        startGridWithConsistentId("C");
+
+        nodeA.active(true);
+
+        stopGrid("C", false);
+
+        nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+        startGridWithConsistentId("D");
+
+        stopAllGrids(false);
+
+        Ignite nodeD = startGridWithConsistentId("D");
+
+        verifyBaselineTopologyHistoryOnNodes(verifier, new Ignite[] {nodeD});
+    }
+
+    /**
+     *
+     */
+    public void testBaselineHistorySyncWithOldNodeWithCompatibleHistory() throws Exception {
+        final long expectedBranchingHash0 = "A".hashCode() + "B".hashCode() + "C".hashCode();
+
+        final long expectedBranchingHash1 = "A".hashCode() + "B".hashCode();
+
+        BaselineTopologyHistoryVerifier verifier = new BaselineTopologyHistoryVerifier() {
+            @Override public void verify(BaselineTopologyHistory bltHist) {
+                assertNotNull(bltHist);
+
+                assertEquals(2, bltHist.history().size());
+
+                BaselineTopologyHistoryItem histItem = bltHist.history().get(0);
+
+                assertEquals(1, histItem.branchingHistory().size());
+
+                long actualBranchingHash0 = histItem.branchingHistory().get(0);
+
+                assertEquals(expectedBranchingHash0, actualBranchingHash0);
+
+                histItem = bltHist.history().get(1);
+
+                assertEquals(1, histItem.branchingHistory().size());
+
+                long actualBranchingHash1 = histItem.branchingHistory().get(0);
+
+                assertEquals(expectedBranchingHash1, actualBranchingHash1);
+            }
+        };
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        startGridWithConsistentId("C");
+
+        nodeA.active(true);
+
+        stopGrid("C", false);
+
+        nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+        stopGrid("B", false);
+
+        nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+        startGridWithConsistentId("B");
+
+        stopAllGrids(false);
+
+        startGridWithConsistentId("A");
+        Ignite nodeB = startGridWithConsistentId("B");
+
+        verifyBaselineTopologyHistoryOnNodes(verifier, new Ignite[] {nodeB});
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testBaselineNotDeletedOnDeactivation() throws Exception {
+        Ignite nodeA = startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        startGridWithConsistentId("C");
+
+        nodeA.active(true);
+
+        assertNotNull(nodeA.cluster().currentBaselineTopology());
+
+        nodeA.active(false);
+
+        stopAllGrids();
+
+        nodeA = startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        startGridWithConsistentId("C");
+
+        final Ignite ig = nodeA;
+
+        boolean clusterActive = GridTestUtils.waitForCondition(
+            new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    return ig.active();
+                }
+            },
+            10_000);
+
+        assertNotNull(nodeA.cluster().currentBaselineTopology());
+
+        assertTrue(clusterActive);
+    }
+
+    /**
+     *
+     */
+    public void testNodeWithBltIsProhibitedToJoinNewCluster() throws Exception {
+        BaselineTopologyVerifier nullVerifier = new BaselineTopologyVerifier() {
+            @Override public void verify(BaselineTopology blt) {
+                assertNull(blt);
+            }
+        };
+
+        Ignite nodeC = startGridWithConsistentId("C");
+
+        nodeC.active(true);
+
+        stopGrid("C", false);
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        Ignite nodeB = startGridWithConsistentId("B");
+
+        verifyBaselineTopologyOnNodes(nullVerifier, new Ignite[] {nodeA, nodeB});
+
+        boolean expectedExceptionThrown = false;
+
+        try {
+            startGridWithConsistentId("C");
+        }
+        catch (IgniteCheckedException e) {
+            expectedExceptionThrown = true;
+
+            if (e.getCause() != null && e.getCause().getCause() != null) {
+                Throwable rootCause = e.getCause().getCause();
+
+                if (!(rootCause instanceof IgniteSpiException) || !rootCause.getMessage().contains("Node with set up BaselineTopology"))
+                    Assert.fail("Unexpected ignite exception was thrown: " + e);
+            }
+            else
+                throw e;
+        }
+
+        assertTrue("Expected exception wasn't thrown.", expectedExceptionThrown);
+    }
+
+    /**
+     * Restore this test when requirements for BaselineTopology deletion are clarified and this feature
+     * is covered with more tests.
+     */
+    public void _testBaselineTopologyHistoryIsDeletedOnBaselineDelete() throws Exception {
+        BaselineTopologyHistoryVerifier verifier = new BaselineTopologyHistoryVerifier() {
+            @Override public void verify(BaselineTopologyHistory bltHist) {
+                assertNotNull(bltHist);
+
+                assertEquals(0, bltHist.history().size());
+            }
+        };
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        startGridWithConsistentId("C");
+
+        nodeA.active(true);
+
+        stopAllGrids(false);
+
+        nodeA = startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+
+        nodeA.active(true);
+
+        nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+        stopAllGrids(false);
+
+        nodeA = startGridWithConsistentId("A");
+
+        nodeA.active(true);
+
+        nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+        stopAllGrids(false);
+
+        final Ignite node = startGridWithConsistentId("A");
+
+        boolean activated = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return node.active();
+            }
+        }, 10_000);
+
+        assertTrue(activated);
+
+        node.cluster().setBaselineTopology(null);
+
+        verifyBaselineTopologyHistoryOnNodes(verifier, new Ignite[] {node});
+
+        stopAllGrids(false);
+
+        nodeA = startGridWithConsistentId("A");
+
+        verifyBaselineTopologyHistoryOnNodes(verifier, new Ignite[] {nodeA});
+    }
+
+    /**
+     * Retrieves baseline topology from ignite node instance.
+     *
+     * @param ig Ig.
+     */
+    private BaselineTopology getBaselineTopology(Ignite ig) {
+        return ((DiscoveryDataClusterState) U.field(
+            (Object) U.field(
+                (Object) U.field(ig, "ctx"),
+                "stateProc"),
+            "globalState"))
+            .baselineTopology();
+    }
+
+    /** */
+    private BaselineTopologyHistory getBaselineTopologyHistory(Ignite ig) {
+        return U.field(
+            (Object) U.field(
+                (Object) U.field(ig, "ctx"),
+                "stateProc"),
+            "bltHist");
+    }
+
+    /** */
+    private void checkDataInCache(IgniteEx srv) {
+        IgniteCache<Object, Object> cache = srv.cache(DEFAULT_CACHE_NAME);
+
+        for (int i = 0; i < ENTRIES_COUNT; i++) {
+            TestValue testVal = (TestValue) cache.get(i);
+
+            assertNotNull(testVal);
+
+            assertEquals(i, testVal.id);
+        }
+    }
+
+    /** */
+    private void createAndFillCache(Ignite srv) {
+        IgniteCache cache = srv.getOrCreateCache(cacheConfiguration());
+
+        for (int i = 0; i < ENTRIES_COUNT; i++)
+            cache.put(i, new TestValue(i, "str" + i));
+    }
+
+    /** */
+    private CacheConfiguration cacheConfiguration() {
+        return new CacheConfiguration()
+            .setName(DEFAULT_CACHE_NAME)
+            .setCacheMode(CacheMode.PARTITIONED)
+            .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+            .setBackups(2)
+            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+    }
+
+    /** */
+    private static final class TestValue {
+        /** */
+        private final int id;
+
+        /** */
+        private final String strId;
+
+        /** */
+        private TestValue(int id, String strId) {
+            this.id = id;
+            this.strId = strId;
+        }
+    }
+
+    /** */
+    private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+        Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+        for (ClusterNode clN : clNodes)
+            res.add(clN);
+
+        return res;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java
index 50d7e7e..73c269a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java
@@ -17,6 +17,11 @@
 package org.apache.ignite.internal.processors.cache.persistence;
 
 import java.io.File;
+import java.nio.file.CopyOption;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.util.Arrays;
 import java.util.Collection;
 import org.apache.ignite.Ignite;
@@ -35,6 +40,7 @@ import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -56,12 +62,21 @@ public class IgnitePdsBinaryMetadataOnClusterRestartTest extends GridCommonAbstr
     private static final String DYNAMIC_STR_FIELD_NAME = "strField";
 
     /** */
+    private static final String CUSTOM_WORK_DIR_NAME_PATTERN = "node%s_workDir";
+
+    /** */
     private boolean clientMode;
 
+    /** */
+    private String customWorkSubDir;
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
+        if (customWorkSubDir != null)
+            cfg.setWorkDirectory(Paths.get(U.defaultWorkDirectory(), customWorkSubDir).toString());
+
         cfg.setClientMode(clientMode);
 
         cfg.setDataStorageConfiguration(
@@ -99,6 +114,257 @@ public class IgnitePdsBinaryMetadataOnClusterRestartTest extends GridCommonAbstr
     }
 
     /**
+     * @see <a href="https://issues.apache.org/jira/browse/IGNITE-7258">IGNITE-7258</a> refer to the following JIRA for more context about the problem verified by the test.
+     */
+    public void testUpdatedBinaryMetadataIsPreservedOnJoinToOldCoordinator() throws Exception {
+        Ignite ignite0 = startGridInASeparateWorkDir("A");
+        Ignite ignite1 = startGridInASeparateWorkDir("B");
+
+        ignite0.active(true);
+
+        IgniteCache<Object, Object> cache0 = ignite0.cache(CACHE_NAME);
+
+        BinaryObject bo = ignite0
+            .binary()
+            .builder(DYNAMIC_TYPE_NAME)
+            .setField(DYNAMIC_INT_FIELD_NAME, 10)
+            .build();
+
+        cache0.put(0, bo);
+
+        stopGrid("A");
+
+        IgniteCache<Object, Object> cache1 = ignite1.cache(CACHE_NAME);
+
+        bo = ignite1
+            .binary()
+            .builder(DYNAMIC_TYPE_NAME)
+            .setField(DYNAMIC_INT_FIELD_NAME, 20)
+            .setField(DYNAMIC_STR_FIELD_NAME, "str")
+            .build();
+
+        cache1.put(1, bo);
+
+        stopAllGrids();
+
+        ignite0 = startGridInASeparateWorkDir("A");
+        ignite1 = startGridInASeparateWorkDir("B");
+
+        awaitPartitionMapExchange();
+
+        cache0 = ignite0.cache(CACHE_NAME).withKeepBinary();
+
+        BinaryObject bObj0 = (BinaryObject) cache0.get(0);
+
+        assertEquals(10, (int) bObj0.field("intField"));
+
+        cache1 = ignite1.cache(CACHE_NAME).withKeepBinary();
+
+        BinaryObject bObj1 = (BinaryObject) cache1.get(1);
+
+        assertEquals(20, (int) bObj1.field("intField"));
+        assertEquals("str", bObj1.field("strField"));
+    }
+
+    /**
+     * @see <a href="https://issues.apache.org/jira/browse/IGNITE-7258">IGNITE-7258</a> refer to the following JIRA for more context about the problem verified by the test.
+     */
+    public void testNewBinaryMetadataIsWrittenOnOldCoordinator() throws Exception {
+        Ignite ignite0 = startGridInASeparateWorkDir("A");
+        Ignite ignite1 = startGridInASeparateWorkDir("B");
+
+        ignite0.active(true);
+
+        IgniteCache<Object, Object> cache0 = ignite0.cache(CACHE_NAME);
+
+        BinaryObject bObj0 = ignite0.binary()
+            .builder("DynamicType0").setField("intField", 10).build();
+
+        cache0.put(0, bObj0);
+
+        stopGrid("A");
+
+        IgniteCache<Object, Object> cache1 = ignite1.cache(CACHE_NAME);
+
+        BinaryObject bObj1 = ignite1.binary()
+            .builder("DynamicType1").setField("strField", "str").build();
+
+        cache1.put(1, bObj1);
+
+        stopAllGrids();
+
+        ignite0 = startGridInASeparateWorkDir("A");
+
+        startGridInASeparateWorkDir("B");
+
+        awaitPartitionMapExchange();
+
+        stopGrid("B");
+
+        cache0 = ignite0.cache(CACHE_NAME).withKeepBinary();
+
+        bObj0 = (BinaryObject) cache0.get(0);
+        bObj1 = (BinaryObject) cache0.get(1);
+
+        assertEquals("DynamicType0", binaryTypeName(bObj0));
+        assertEquals("DynamicType1", binaryTypeName(bObj1));
+
+        assertEquals(10, (int) bObj0.field(DYNAMIC_INT_FIELD_NAME));
+        assertEquals("str", bObj1.field(DYNAMIC_STR_FIELD_NAME));
+    }
+
+    /**
+     * Verifies that newer BinaryMetadata is applied not only on coordinator but on all nodes
+     * with outdated version.
+     *
+     * In the following test nodeB was offline when BinaryMetadata was updated,
+     * after full cluster restart it starts second (so it doesn't take the role of coordinator)
+     * but metadata update is propagated to it anyway.
+     *
+     * @see <a href="https://issues.apache.org/jira/browse/IGNITE-7258">IGNITE-7258</a> refer to the following JIRA for more context about the problem verified by the test.
+     */
+    public void testNewBinaryMetadataIsPropagatedToAllOutOfDataNodes() throws Exception {
+        Ignite igniteA = startGridInASeparateWorkDir("A");
+        startGridInASeparateWorkDir("B");
+        Ignite igniteC = startGridInASeparateWorkDir("C");
+        startGridInASeparateWorkDir("D");
+
+        igniteA.active(true);
+
+        BinaryObject bObj0 = igniteA.binary()
+            .builder(DYNAMIC_TYPE_NAME).setField(DYNAMIC_INT_FIELD_NAME, 10).build();
+
+        igniteA.cache(CACHE_NAME).put(0, bObj0);
+
+        stopGrid("A");
+        stopGrid("B");
+
+        BinaryObject bObj1 = igniteC.binary()
+            .builder(DYNAMIC_TYPE_NAME)
+            .setField(DYNAMIC_INT_FIELD_NAME, 20)
+            .setField(DYNAMIC_STR_FIELD_NAME, "str").build();
+
+        igniteC.cache(CACHE_NAME).put(1, bObj1);
+
+        //full cluster restart
+        stopAllGrids();
+
+        //node A becomes a coordinator
+        igniteA = startGridInASeparateWorkDir("A");
+        //node B isn't a coordinator, but metadata update is propagated to if
+        startGridInASeparateWorkDir("B");
+        //these two nodes will provide an updated version of metadata when started
+        startGridInASeparateWorkDir("C");
+        startGridInASeparateWorkDir("D");
+
+        awaitPartitionMapExchange();
+
+        //stopping everything to make sure that nodeB will lose its in-memory BinaryMetadata cache
+        // and will have to reload it from FS when restarted later
+        stopAllGrids();
+
+        Ignite igniteB = startGridInASeparateWorkDir("B");
+
+        igniteB.active(true);
+
+        bObj1 = (BinaryObject) igniteB.cache(CACHE_NAME).withKeepBinary().get(1);
+
+        assertEquals(20, (int) bObj1.field(DYNAMIC_INT_FIELD_NAME));
+        assertEquals("str", bObj1.field(DYNAMIC_STR_FIELD_NAME));
+    }
+
+    /** */
+    private Ignite startGridInASeparateWorkDir(String nodeName) throws Exception {
+        customWorkSubDir = String.format(CUSTOM_WORK_DIR_NAME_PATTERN, nodeName);
+        return startGrid(nodeName);
+    }
+
+    /** */
+    private String binaryTypeName(BinaryObject bObj) {
+        return bObj.type().typeName();
+    }
+
+    /**
+     * If joining node has incompatible BinaryMetadata (e.g. when user manually copies binary_meta file),
+     * coordinator detects it and fails the node providing information about conflict.
+     *
+     * @see <a href="https://issues.apache.org/jira/browse/IGNITE-7258">IGNITE-7258</a> refer to the following JIRA for more context about the problem verified by the test.
+     */
+    public void testNodeWithIncompatibleMetadataIsProhibitedToJoinTheCluster() throws Exception {
+        final String decimalFieldName = "decField";
+
+        Ignite igniteA = startGridInASeparateWorkDir("A");
+        Ignite igniteB = startGridInASeparateWorkDir("B");
+
+        String bConsId = igniteB.cluster().localNode().consistentId().toString();
+
+        igniteA.active(true);
+
+        IgniteCache<Object, Object> cache = igniteA.cache(CACHE_NAME);
+
+        BinaryObject bObj = igniteA.binary()
+            .builder(DYNAMIC_TYPE_NAME).setField(decimalFieldName, 10).build();
+
+        cache.put(0, bObj);
+
+        stopAllGrids();
+
+        Ignite igniteC = startGridInASeparateWorkDir("C");
+        String cConsId = igniteC.cluster().localNode().consistentId().toString();
+        igniteC.active(true);
+
+        cache = igniteC.cache(CACHE_NAME);
+        bObj = igniteC.binary().builder(DYNAMIC_TYPE_NAME).setField(decimalFieldName, 10L).build();
+
+        cache.put(0, bObj);
+
+        stopAllGrids();
+
+        copyIncompatibleBinaryMetadata(
+            String.format(CUSTOM_WORK_DIR_NAME_PATTERN, "C"),
+            cConsId,
+            String.format(CUSTOM_WORK_DIR_NAME_PATTERN, "B"),
+            bConsId,
+            DYNAMIC_TYPE_NAME.toLowerCase().hashCode() + ".bin");
+
+        startGridInASeparateWorkDir("A");
+
+        boolean exceptedExceptionThrown = false;
+        try {
+            startGridInASeparateWorkDir("B");
+        }
+        catch (Exception e) {
+            if (e.getCause() != null && e.getCause().getCause() != null) {
+                if (e.getCause().getCause().getMessage().contains(
+                        String.format("[typeName=%s, fieldName=%s, fieldTypeName1=int, fieldTypeName2=long]",
+                            DYNAMIC_TYPE_NAME,
+                            decimalFieldName)
+                ))
+                    exceptedExceptionThrown = true;
+            }
+            else
+                throw e;
+        }
+
+        assertTrue(exceptedExceptionThrown);
+    }
+
+    /** */
+    private void copyIncompatibleBinaryMetadata(String fromWorkDir,
+        String fromConsId,
+        String toWorkDir,
+        String toConsId,
+        String fileName
+    ) throws Exception {
+        String workDir = U.defaultWorkDirectory();
+
+        Path fromFile = Paths.get(workDir, fromWorkDir, "binary_meta", "node00-" + fromConsId, fileName);
+        Path toFile = Paths.get(workDir, toWorkDir, "binary_meta", "node00-" + toConsId, fileName);
+
+        Files.copy(fromFile, toFile, StandardCopyOption.REPLACE_EXISTING);
+    }
+
+    /**
      * Test verifies that binary metadata from regular java classes is saved and restored correctly
      * on cluster restart.
      */
@@ -237,7 +503,7 @@ public class IgnitePdsBinaryMetadataOnClusterRestartTest extends GridCommonAbstr
 
         awaitPartitionMapExchange();
 
-        examineDynamicMetadata(4, contentExaminer0, contentExaminer1, structureExaminer1);
+        examineDynamicMetadata(2, contentExaminer0, contentExaminer1, structureExaminer1);
     }
 
     /**
@@ -350,6 +616,8 @@ public class IgnitePdsBinaryMetadataOnClusterRestartTest extends GridCommonAbstr
         stopAllGrids();
 
         cleanIgniteWorkDir();
+
+        customWorkSubDir = null;
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
index 8e43e93..bf20ee4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
@@ -686,7 +686,12 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
     private static class CoordinatorNodeFilter implements IgnitePredicate<ClusterNode> {
         /** {@inheritDoc} */
         @Override public boolean apply(ClusterNode node) {
-            return node.order() > 1;
+            try {
+                return node.order() > 1;
+            }
+            catch (UnsupportedOperationException e) {
+                return false;
+            }
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
index e9828f5..301c0bc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
@@ -56,6 +56,8 @@ public class IgnitePersistentStoreDataStructuresTest extends GridCommonAbstractT
 
         cfg.setDataStorageConfiguration(memCfg);
 
+        cfg.setAutoActivationEnabled(false);
+
         return cfg;
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAllBaselineNodesOnlineFullApiSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAllBaselineNodesOnlineFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAllBaselineNodesOnlineFullApiSelfTest.java
new file mode 100644
index 0000000..c6aed7d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAllBaselineNodesOnlineFullApiSelfTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteAllBaselineNodesOnlineFullApiSelfTest extends IgniteBaselineAbstractFullApiSelfTest {
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        initStoreStrategy();
+
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(gridCount());
+
+        grid(0).active(true);
+
+        awaitPartitionMapExchange();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java
new file mode 100644
index 0000000..8dcfc0b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest;
+
+/**
+ *
+ */
+public abstract class IgniteBaselineAbstractFullApiSelfTest extends GridCacheAbstractFullApiSelfTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setMaxSize(200 * 1024 * 1024)
+                    .setPersistenceEnabled(true)));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected int gridCount() {
+        return 4;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getPartitionMapExchangeTimeout() {
+        return 60_000;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCachePutAllFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCachePutAllFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCachePutAllFailoverTest.java
new file mode 100644
index 0000000..16fa971
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCachePutAllFailoverTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.distributed.CachePutAllFailoverAbstractTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Failover test for cache putAll operations when BaselineTopology is changed down
+ * (existing node gets stopped and removed from BaselineTopology).
+ */
+public class IgniteChangingBaselineDownCachePutAllFailoverTest extends CachePutAllFailoverAbstractTest {
+    /** */
+    private static final int GRIDS_COUNT = 5;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                        .setInitialSize(200 * 1024 * 1024)
+                        .setMaxSize(200 * 1024 * 1024)
+                        .setCheckpointPageBufferSize(200 * 1024 * 1024)
+                )
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(GRIDS_COUNT);
+
+        grid(0).active(true);
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean finished, final long endTime) {
+        return GridTestUtils.runAsync(new Callable() {
+            @Override public Object call() throws Exception {
+                Thread.currentThread().setName("restart-thread");
+
+                U.sleep(15_000);
+
+                ThreadLocalRandom tlr = ThreadLocalRandom.current();
+
+                int idx = tlr.nextInt(1, GRIDS_COUNT);
+
+                log.info("Stopping node " + idx);
+
+                stopGrid(idx);
+
+                IgniteEx ig0 = grid(0);
+
+                ig0.cluster().setBaselineTopology(baselineNodes(ig0.cluster().forServers().nodes()));
+
+                U.sleep(3_000);
+
+                return null;
+            }
+        });
+    }
+
+    /** */
+    private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+        Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+        for (ClusterNode clN : clNodes)
+            res.add(clN);
+
+        return res;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCacheRemoveFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCacheRemoveFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCacheRemoveFailoverTest.java
new file mode 100644
index 0000000..86b5809
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCacheRemoveFailoverTest.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractRemoveFailureTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
+/**
+ * Failover test for cache remove operations when BaselineTopology is changed down
+ * (existing node gets stopped and removed from BaselineTopology).
+ */
+public class IgniteChangingBaselineDownCacheRemoveFailoverTest extends GridCacheAbstractRemoveFailureTest {
+    /** */
+    private static final int GRIDS_COUNT = 5;
+
+    /** {@inheritDoc} */
+    @Override protected CacheMode cacheMode() {
+        return PARTITIONED;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected CacheAtomicityMode atomicityMode() {
+        return TRANSACTIONAL;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected NearCacheConfiguration nearCache() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                    .setPersistenceEnabled(true)
+                    .setInitialSize(200 * 1024 * 1024)
+                    .setMaxSize(200 * 1024 * 1024)
+                    .setCheckpointPageBufferSize(200 * 1024 * 1024)
+                )
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(GRIDS_COUNT);
+
+        startGrid(GRIDS_COUNT);
+
+        grid(0).active(true);
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean stop, final AtomicReference<CyclicBarrier> cmp) {
+        return GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                Thread.currentThread().setName("restart-thread");
+
+                U.sleep(random(5000, 10_000));
+
+                log.info("Stopping node " + GRIDS_COUNT);
+
+                stopGrid(GRIDS_COUNT);
+
+                IgniteEx ig0 = grid(0);
+
+                ig0.cluster().setBaselineTopology(baselineNodes(ig0.cluster().forServers().nodes()));
+
+                while (!stop.get()) {
+                    CyclicBarrier barrier = cmp.get();
+
+                    if (barrier != null) {
+                        log.info("Wait data check.");
+
+                        barrier.await(60_000, TimeUnit.MILLISECONDS);
+
+                        log.info("Finished wait data check.");
+                    }
+                }
+
+                return null;
+            }
+        });
+    }
+
+    /** */
+    private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+        Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+        for (ClusterNode clN : clNodes)
+            res.add(clN);
+
+        return res;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCachePutAllFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCachePutAllFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCachePutAllFailoverTest.java
new file mode 100644
index 0000000..44c8402
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCachePutAllFailoverTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.distributed.CachePutAllFailoverAbstractTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Failover test for cache putAll operations when BaselineTopology is changed up
+ * (new node is added to BaselineTopology).
+ */
+public class IgniteChangingBaselineUpCachePutAllFailoverTest extends CachePutAllFailoverAbstractTest {
+    /** */
+    private static final int GRIDS_COUNT = 5;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                        .setInitialSize(200 * 1024 * 1024)
+                        .setMaxSize(200 * 1024 * 1024)
+                        .setCheckpointPageBufferSize(200 * 1024 * 1024)
+                )
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(GRIDS_COUNT);
+
+        grid(0).active(true);
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean finished, final long endTime) {
+        return GridTestUtils.runAsync(new Callable() {
+            @Override public Object call() throws Exception {
+                Thread.currentThread().setName("restart-thread");
+
+                U.sleep(15_000);
+
+                log.info("Starting node");
+
+                startGrid(GRIDS_COUNT);
+
+                IgniteEx ig0 = grid(0);
+
+                ig0.cluster().setBaselineTopology(baselineNodes(ig0.cluster().forServers().nodes()));
+
+                U.sleep(3_000);
+
+                return null;
+            }
+        });
+    }
+
+    /** */
+    private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+        Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+        for (ClusterNode clN : clNodes)
+            res.add(clN);
+
+        return res;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCacheRemoveFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCacheRemoveFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCacheRemoveFailoverTest.java
new file mode 100644
index 0000000..8391e0c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCacheRemoveFailoverTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractRemoveFailureTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
+/**
+ * Failover test for cache remove operations when BaselineTopology is changed up
+ * (new node is added to BaselineTopology).
+ */
+public class IgniteChangingBaselineUpCacheRemoveFailoverTest extends GridCacheAbstractRemoveFailureTest {
+    /** */
+    private static final int GRIDS_COUNT = 5;
+
+    /** {@inheritDoc} */
+    @Override protected CacheMode cacheMode() {
+        return PARTITIONED;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected CacheAtomicityMode atomicityMode() {
+        return TRANSACTIONAL;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected NearCacheConfiguration nearCache() {
+        return null;
+    }
+
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                    .setPersistenceEnabled(true)
+                    .setInitialSize(200 * 1024 * 1024)
+                    .setMaxSize(200 * 1024 * 1024)
+                    .setCheckpointPageBufferSize(200 * 1024 * 1024)
+                )
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(GRIDS_COUNT);
+
+        grid(0).active(true);
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean stop, final AtomicReference<CyclicBarrier> cmp) {
+        return GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                Thread.currentThread().setName("restart-thread");
+
+                U.sleep(random(5000, 10_000));
+
+                log.info("Starting node " + GRIDS_COUNT);
+
+                startGrid(GRIDS_COUNT);
+
+                IgniteEx ig0 = grid(0);
+
+                ig0.cluster().setBaselineTopology(baselineNodes(ig0.cluster().forServers().nodes()));
+
+                while (!stop.get()) {
+                    CyclicBarrier barrier = cmp.get();
+
+                    if (barrier != null) {
+                        log.info("Wait data check.");
+
+                        barrier.await(60_000, TimeUnit.MILLISECONDS);
+
+                        log.info("Finished wait data check.");
+                    }
+                }
+
+                return null;
+            }
+        });
+    }
+
+    /** */
+    private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+        Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+        for (ClusterNode clN : clNodes)
+            res.add(clN);
+
+        return res;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOfflineBaselineNodeFullApiSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOfflineBaselineNodeFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOfflineBaselineNodeFullApiSelfTest.java
new file mode 100644
index 0000000..3fd4974
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOfflineBaselineNodeFullApiSelfTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteOfflineBaselineNodeFullApiSelfTest extends IgniteBaselineAbstractFullApiSelfTest {
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        initStoreStrategy();
+
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(gridCount());
+
+        startGrid("offlineBaselineNode");
+
+        grid(0).active(true);
+
+        stopGrid("offlineBaselineNode");
+
+        awaitPartitionMapExchange();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOnlineNodeOutOfBaselineFullApiSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOnlineNodeOutOfBaselineFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOnlineNodeOutOfBaselineFullApiSelfTest.java
new file mode 100644
index 0000000..07c475b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOnlineNodeOutOfBaselineFullApiSelfTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteOnlineNodeOutOfBaselineFullApiSelfTest extends IgniteBaselineAbstractFullApiSelfTest {
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        initStoreStrategy();
+
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(gridCount());
+
+        grid(0).active(true);
+
+        startGrid("onlineOutOfBaselineNode");
+
+        awaitPartitionMapExchange();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCachePutAllFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCachePutAllFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCachePutAllFailoverTest.java
new file mode 100644
index 0000000..13f3489
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCachePutAllFailoverTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.distributed.CachePutAllFailoverAbstractTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteStableBaselineCachePutAllFailoverTest extends CachePutAllFailoverAbstractTest {
+    /** */
+    private static final int GRIDS_COUNT = 3;
+
+    /** */
+    private static final int OUT_OF_BASELINE_GRID_ID = GRIDS_COUNT;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                        .setInitialSize(200 * 1024 * 1024)
+                        .setMaxSize(200 * 1024 * 1024)
+                    .setCheckpointPageBufferSize(200 * 1024 * 1024)
+                )
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean finished, final long endTime) {
+        return GridTestUtils.runAsync(new Callable() {
+            @Override public Object call() throws Exception {
+                Thread.currentThread().setName("restart-thread");
+
+                while (!finished.get() && System.currentTimeMillis() < endTime) {
+                    ThreadLocalRandom tlr = ThreadLocalRandom.current();
+
+                    int idx = tlr.nextInt(1, GRIDS_COUNT + 1);
+
+                    log.info("Stopping node " + idx);
+
+                    stopGrid(idx);
+
+                    U.sleep(tlr.nextInt(500, 700));
+
+                    log.info("Restarting node " + idx);
+
+                    startGrid(idx);
+                }
+
+                return null;
+            }
+        });
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(GRIDS_COUNT);
+
+        grid(0).active(true);
+
+        startGrid(OUT_OF_BASELINE_GRID_ID);
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+}