You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2018/01/17 10:35:13 UTC

[01/17] ignite git commit: IEP-4 Baseline topology for persistent caches (Phase 1) Contributed by: Dmitriy Govorukhin Dmitry Pavlov Eduard Shangareev Ily

Repository: ignite
Updated Branches:
  refs/heads/master 43a62cdba -> 6f7aba852


http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
----------------------------------------------------------------------
diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
index f3268d9..2da1aa3 100644
--- a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
+++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.development.utils;
 
 import java.io.File;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.internal.pagemem.wal.WALIterator;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
@@ -30,6 +31,7 @@ import org.apache.ignite.internal.processors.query.h2.database.io.H2InnerIO;
 import org.apache.ignite.internal.processors.query.h2.database.io.H2LeafIO;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.logger.NullLogger;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Print WAL log data in human-readable form.
@@ -41,7 +43,7 @@ public class IgniteWalConverter {
     public static void main(String[] args) throws Exception {
         if (args.length < 2)
             throw new IllegalArgumentException("\nYou need to provide:\n" +
-                    "\t1. Size of pages (1024, 2048, etc).\n" +
+                    "\t1. Size of pages, which was selected for file store (1024, 2048, 4096, etc).\n" +
                     "\t2. Path to dir with wal files.\n" +
                     "\t3. (Optional) Path to dir with archive wal files.");
 
@@ -49,6 +51,9 @@ public class IgniteWalConverter {
         H2ExtrasInnerIO.register();
         H2ExtrasLeafIO.register();
 
+        boolean printRecords = IgniteSystemProperties.getBoolean("PRINT_RECORDS", false);
+        boolean printStat = IgniteSystemProperties.getBoolean("PRINT_STAT", true);
+
         final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(new NullLogger(),
             Integer.parseInt(args[0]),
             null,
@@ -62,11 +67,20 @@ public class IgniteWalConverter {
         if (workFiles == null)
             throw new IllegalArgumentException("No .wal files in dir: " + args[1]);
 
+        @Nullable final WalStat stat = printStat ? new WalStat() : null;
+
         try (WALIterator stIt = factory.iteratorWorkFiles(workFiles)) {
             while (stIt.hasNextX()) {
                 IgniteBiTuple<WALPointer, WALRecord> next = stIt.nextX();
 
-                System.out.println("[W] " + next.get2());
+                final WALPointer pointer = next.get1();
+                final WALRecord record = next.get2();
+
+                if (stat != null)
+                    stat.registerRecord(record, pointer, true);
+
+                if (printRecords)
+                    System.out.println("[W] " + record);
             }
         }
 
@@ -77,9 +91,21 @@ public class IgniteWalConverter {
                 while (stIt.hasNextX()) {
                     IgniteBiTuple<WALPointer, WALRecord> next = stIt.nextX();
 
-                    System.out.println("[A] " + next.get2());
+                    final WALPointer pointer = next.get1();
+                    final WALRecord record = next.get2();
+
+                    if (stat != null)
+                        stat.registerRecord(record, pointer, false);
+
+                    if (printRecords)
+                        System.out.println("[A] " + record);
                 }
             }
         }
+
+        System.err.flush();
+
+        if (stat != null)
+            System.out.println("Statistic collected:\n" + stat.toString());
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/RecordSizeCountStat.java
----------------------------------------------------------------------
diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/RecordSizeCountStat.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/RecordSizeCountStat.java
new file mode 100644
index 0000000..1d0a1c0
--- /dev/null
+++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/RecordSizeCountStat.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.development.utils;
+
+/**
+ * Statistic for record size, used to accumulate information about several record and calculating average.
+ */
+public class RecordSizeCountStat {
+    /** Sum of sizes. */
+    private long size = -1;
+
+    /** Count of all records. */
+    private int cnt;
+
+    /**
+     * @param size record size. Negative value means size is unknown for current occurrence. Any negative value resets
+     * accumulated statistics.
+     */
+    void occurrence(int size) {
+        if (size >= 0) {
+            if (this.size < 0)
+                this.size = 0;
+
+            this.size += size;
+        }
+        else {
+            if (this.size > 0)
+                this.size = -1;
+        }
+
+        cnt++;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return (size >= 0 ? size : "") +
+            "\t " + cnt +
+            "\t " + ((size >= 0) ? size / cnt : "");
+    }
+
+    /**
+     * @return average size of one record, as double.
+     */
+    private double averageD() {
+        return 1.0 * size / cnt;
+    }
+
+    /**
+     * @return average size of one record, printable version.
+     */
+    String averageStr() {
+        return cnt == 0 ? "" : String.format("%.2f", averageD());
+    }
+
+    /**
+     * @return Count of all records.
+     */
+    int getCount() {
+        return cnt;
+    }
+
+    /**
+     * @param val other statistic value to reduce with.
+     */
+    void add(RecordSizeCountStat val) {
+        cnt += val.cnt;
+        size += val.size;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/TxWalStat.java
----------------------------------------------------------------------
diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/TxWalStat.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/TxWalStat.java
new file mode 100644
index 0000000..a45b8ea
--- /dev/null
+++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/TxWalStat.java
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.development.utils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+
+/**
+ * Transactions statistics for WAL.
+ */
+public class TxWalStat {
+    /** Maximum element in popular combinations discovery map before eviction. */
+    private static final int POPULAR_COMBINATION_MAP_MAX_SIZE = 50000;
+
+    /** Usages count, when evict prohibited from popular combinations discovery map. */
+    private static final int USAGES_CNT_EVICT_PROHIBITED = 10;
+
+    /** Transactions in preparing/committing state. When commit is finished, TX is removed from this collection. */
+    private final Map<GridCacheVersion, TxOccurrence> opened = new HashMap<>();
+
+    /** Field for calculating average number of primary nodes involved in Tx. */
+    private final RecordSizeCountStat avgPrimaryNodes = new RecordSizeCountStat();
+
+    /** Field for calculating average number of total nodes involved in Tx. */
+    private final RecordSizeCountStat avgTotalNodes = new RecordSizeCountStat();
+
+    /** Tx statistics: Entries updated count -> count of such Txes. */
+    private final Map<Integer, Integer> txEntriesUpdated = new TreeMap<>();
+
+    /** Tx statistics: Caches Involved  count -> count of such Txes.  */
+    private final Map<Integer, Integer> txCachesInvolved = new TreeMap<>();
+
+    /** Cache IDs combination involved in Tx. Popular combination search map, with possible eviction. */
+    private final LruMap<String> cacheIdsInTx = new LruMap<>(POPULAR_COMBINATION_MAP_MAX_SIZE,
+        USAGES_CNT_EVICT_PROHIBITED);
+
+    /**
+     * Cache IDs combination involved in Tx, weighted using primary nodes in Tx. Used to search popular combinations
+     * mostly involved into highly distributive transactions.
+     */
+    private final LruMap<String> cacheIdsWeightedNodesInTx = new LruMap<>(POPULAR_COMBINATION_MAP_MAX_SIZE,
+        USAGES_CNT_EVICT_PROHIBITED * 90);
+    /**
+     * Cache IDs combination involved in Tx, weighted using total nodes in Tx. Used to search popular combinations
+     * mostly involved into highly distributive transactions.
+     */
+    private final LruMap<String> cacheIdsWeightedTotalNodesInTx = new LruMap<>(POPULAR_COMBINATION_MAP_MAX_SIZE,
+        USAGES_CNT_EVICT_PROHIBITED * 150);
+
+    /**
+     * @param key key (parameter) value found.
+     * @param map map to save increment.
+     * @param <K> type of key.
+     */
+    private static <K> void incrementStat(K key, Map<K, Integer> map) {
+        incrementStat(key, map, 1);
+    }
+
+    /**
+     * @param key key (parameter) value found.
+     * @param map map to save increment.
+     * @param increment value to increment statistic, 1 or weight of current occurrence.
+     * @param <K> type of key.
+     */
+    private static <K> void incrementStat(K key, Map<K, Integer> map, int increment) {
+        Integer val = map.get(key);
+        int recordStat = val == null ? 0 : val;
+
+        recordStat += increment;
+        map.put(key, recordStat);
+    }
+
+    /**
+     * Handles TX prepare: creates TX in {@link #opened} map.
+     * @param nearXidVer near Xid Version. Global transaction identifier within cluster, assigned by transaction
+     * coordinator.
+     * @param nodes primary nodes registered in prepare record.
+     * @param totalNodes all nodes (primary & backup) in prepare record.
+     */
+    void onTxPrepareStart(GridCacheVersion nearXidVer, int nodes, int totalNodes) {
+        txComputeIfAbsent(nearXidVer, nodes, totalNodes);
+    }
+
+    /**
+     * @param nearXidVer near Xid Version. Global transaction identifier within cluster, assigned by transaction
+     * coordinator.
+     * @param nodes primary nodes registered in prepare record.
+     * @param totalNodes all nodes (primary & backup) in prepare record.
+     * @return tx occurrence to accumulate entries into.
+     */
+    private TxOccurrence txComputeIfAbsent(GridCacheVersion nearXidVer, int nodes, int totalNodes) {
+        TxOccurrence occurrence = opened.get(nearXidVer);
+
+        if (occurrence == null)
+            occurrence = new TxOccurrence(nodes, totalNodes);
+
+        opened.put(nearXidVer, occurrence);
+
+        return occurrence;
+    }
+
+    /**
+     * Handles commit or rollback transaction. Finished statistics accumulation.
+     *
+     * @param nearXidVer near Xid Version. Global transaction identifier within cluster, assigned by transaction
+     * coordinator.
+     * @param commit tx committed, flag indicating TX successes.
+     */
+    void onTxEnd(GridCacheVersion nearXidVer, boolean commit) {
+        TxOccurrence occurrence = opened.remove(nearXidVer);
+
+        if (occurrence == null)
+            return;
+
+        if (!commit)
+            return;
+
+        if (occurrence.nodes > 0 && occurrence.totalNodes > 0) {
+            avgPrimaryNodes.occurrence(occurrence.nodes);
+            avgTotalNodes.occurrence(occurrence.totalNodes);
+        }
+
+        incrementStat(occurrence.entriesUpdated, txEntriesUpdated);
+
+        incrementStat(occurrence.caches.size(), txCachesInvolved);
+
+        if (!occurrence.caches.isEmpty()) {
+            final String sortedCachesKey = occurrence.caches.toString();
+
+            incrementStat(sortedCachesKey, cacheIdsInTx.map, 1);
+
+            if (occurrence.nodes > 0)
+                incrementStat(sortedCachesKey, cacheIdsWeightedNodesInTx.map, occurrence.nodes);
+
+            if (occurrence.totalNodes > 0)
+                incrementStat(sortedCachesKey, cacheIdsWeightedTotalNodesInTx.map, occurrence.totalNodes);
+        }
+    }
+
+    /**
+     * Handles Data entry from data record. Entries not under transaction are ignored.
+     *
+     * @param entry object updated.
+     */
+    void onDataEntry(DataEntry entry) {
+        final GridCacheVersion ver = entry.nearXidVersion();
+
+        if (ver == null)
+            return;
+
+        txComputeIfAbsent(ver, -1, -1).onDataEntry(entry);
+    }
+
+    /**
+     * @param sb buffer.
+     * @param mapName display name of map.
+     * @param map values.
+     */
+    private void printSizeCountMap(StringBuilder sb, String mapName, Map<?, Integer> map) {
+        sb.append(mapName).append(": \n");
+        sb.append("key\tcount");
+        sb.append("\n");
+
+        final List<? extends Map.Entry<?, Integer>> entries = new ArrayList<>(map.entrySet());
+
+        Collections.sort(entries, new Comparator<Map.Entry<?, Integer>>() {
+            @Override public int compare(Map.Entry<?, Integer> o1, Map.Entry<?, Integer> o2) {
+                return -Integer.compare(o1.getValue(), o2.getValue());
+            }
+        });
+
+        int othersCnt = 0;
+        int othersSum = 0;
+        int cnt = 0;
+
+        for (Map.Entry<?, Integer> next : entries) {
+            if (cnt < WalStat.DISPLAY_MAX) {
+                sb.append(next.getKey()).append("\t").append(next.getValue()).append("\t");
+                sb.append("\n");
+            }
+            else {
+                othersCnt++;
+                othersSum += next.getValue();
+            }
+            cnt++;
+        }
+
+        if (othersCnt > 0) {
+            sb.append("... other ").append(othersCnt).append(" values").append("\t").append(othersSum).append("\t");
+            sb.append("\n");
+        }
+
+        sb.append("\n");
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        final StringBuilder sb = new StringBuilder();
+        sb.append("Tx stat: remained Opened: \t").append(opened.size()).append("\n").append("\n");
+
+        sb.append("Tx stat: Average Primary Node Count: \t").append(avgPrimaryNodes.averageStr()).append("\n").append("\n");
+
+        sb.append("Tx stat: Average Total Node Count: \t").append(avgTotalNodes.averageStr()).append("\n").append("\n");
+
+        printSizeCountMap(sb, "Tx stat: Entries updated", txEntriesUpdated);
+        printSizeCountMap(sb, "Tx stat: Caches involved", txCachesInvolved);
+        printSizeCountMap(sb, "Tx stat: Caches list in TX, evicted = " + cacheIdsInTx.evicted, cacheIdsInTx.map);
+
+        printSizeCountMap(sb, "Tx stat: Caches list in TX; weighted by primary Nodes, evicted = "
+            + cacheIdsWeightedNodesInTx.evicted, cacheIdsWeightedNodesInTx.map);
+
+        printSizeCountMap(sb, "Tx stat: Caches list in TX; weighted by total Nodes, evicted = "
+            + cacheIdsWeightedNodesInTx.evicted, cacheIdsWeightedNodesInTx.map);
+        return sb.toString();
+    }
+
+    /**
+     * Tx in prepare or in commit state, used to accumulate statistic.
+     */
+    private static class TxOccurrence {
+        /** Primary nodes count from TX record. */
+        private int nodes;
+        /** Primary + backup nodes count from TX record. */
+        private int totalNodes;
+
+        /** Count of entries updated under current transaction on current node. */
+        private int entriesUpdated;
+
+        /** Sorted set of cache IDs updated during this transaction. */
+        private TreeSet<Integer> caches = new TreeSet<>();
+
+        /**
+         * @param nodes Primary nodes count from TX record.
+         * @param totalNodes Primary + backup nodes count from TX record.
+         */
+        TxOccurrence(int nodes, int totalNodes) {
+            this.nodes = nodes;
+            this.totalNodes = totalNodes;
+        }
+
+        /**
+         * Handles data entry from data record.
+         * @param entry object updated.
+         */
+        void onDataEntry(DataEntry entry) {
+            entriesUpdated++;
+
+            caches.add(entry.cacheId());
+        }
+    }
+
+    /**
+     * @param <K> key type parameter.
+     */
+    private static class LruMap<K> {
+        /** Max size of map after which eviction may start. */
+        private int maxSize;
+
+        /**
+         * Evict prohibited boundary. If this number of usages is accumulate in eldest entry it will not be removed
+         * anyway.
+         */
+        private int evictProhibited;
+        /**
+         * Evicted count. Number of entries removed during statistic accumulation. Zero value means all records were
+         * processed, created top (popular combination search) is totally correct. Non zero means top may be not
+         * correct.
+         */
+        private int evicted;
+
+        /** Map with data. */
+        private Map<K, Integer> map = new LinkedHashMap<K, Integer>(16, 0.75F, false) {
+            @Override protected boolean removeEldestEntry(Map.Entry<K, Integer> eldest) {
+                if (size() < maxSize)
+                    return false;
+
+                final boolean evictNow = eldest.getValue() < evictProhibited;
+
+                if (evictNow)
+                    evicted++;
+
+                return evictNow;
+            }
+        };
+
+        /**
+         * @param maxSize Max size of map.
+         * @param evictProhibited usages count, when evict became prohibited.
+         */
+        LruMap(int maxSize, int evictProhibited) {
+            this.maxSize = maxSize;
+            this.evictProhibited = evictProhibited;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/WalStat.java
----------------------------------------------------------------------
diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/WalStat.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/WalStat.java
new file mode 100644
index 0000000..36933d8
--- /dev/null
+++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/WalStat.java
@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.development.utils;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
+import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
+import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.transactions.TransactionState;
+import sun.nio.ch.DirectBuffer;
+
+/**
+ * Statistic for overall WAL file
+ */
+public class WalStat {
+    /** Display max: top lines before merging other values */
+    static final int DISPLAY_MAX = 80;
+
+    /** Tx statistics. */
+    private TxWalStat txStat = new TxWalStat();
+
+    /** Segments source: work/archive  -> records loaded count & sizes. */
+    private final Map<String, RecordSizeCountStat> segmentsFolder = new TreeMap<>();
+
+    /** Segments index -> records loaded count & sizes. */
+    private final Map<String, RecordSizeCountStat> segmentsIndexes = new TreeMap<>();
+
+    /** Record type -> its count & sizes. */
+    private final Map<String, RecordSizeCountStat> recTypeSizes = new TreeMap<>();
+
+    /** Data Record: entries count */
+    private final Map<Integer, RecordSizeCountStat> dataRecordEntriesCnt = new TreeMap<>();
+
+    /** Data Record: is under TX */
+    private final Map<Boolean, RecordSizeCountStat> dataRecordUnderTx = new TreeMap<>();
+
+    /** Data Entry: operation performed */
+    private final Map<String, RecordSizeCountStat> dataEntryOperation = new TreeMap<>();
+
+    /** Data Entry: cache groups. */
+    private final Map<Integer, RecordSizeCountStat> dataEntryCacheId = new TreeMap<>();
+
+    /** Tx Record: action */
+    private final Map<String, RecordSizeCountStat> txRecordAct = new TreeMap<>();
+
+    /** Tx Record: participating primary nodes */
+    private final Map<Integer, RecordSizeCountStat> txRecordPrimNodesCnt = new TreeMap<>();
+
+    /** Tx Record: participating primary nodes */
+    private final Map<Integer, RecordSizeCountStat> txRecordNodesCnt = new TreeMap<>();
+
+    /** Page snapshot types. */
+    private final Map<String, RecordSizeCountStat> pageSnapshotTypes = new TreeMap<>();
+
+    /** Page snapshot indexes. */
+    private final Map<String, RecordSizeCountStat> pageSnapshotIndexes = new TreeMap<>();
+
+    /** Page snapshot cache groups. */
+    private final Map<Integer, RecordSizeCountStat> pageSnapshotCacheGrp = new TreeMap<>();
+
+    /** Page snapshot: partition ID. */
+    private final Map<Integer, RecordSizeCountStat> pageSnapshotPartId = new TreeMap<>();
+
+    /**
+     * @param key key (parameter) value.
+     * @param record record corresponding to {@code key}.
+     * @param map map to save statistics.
+     * @param <K> key type.
+     */
+    private static <K> void incrementStat(K key, WALRecord record, Map<K, RecordSizeCountStat> map) {
+        incrementStat(key, map, record.size());
+    }
+
+    /**
+     * @param key key (parameter) value.
+     * @param map  to save statistics.
+     * @param size record size for record corresponding to {@code key}. Negative value of size means size is unknown for current row.
+     * @param <K>  key type.
+     */
+    private static <K> void incrementStat(K key, Map<K, RecordSizeCountStat> map, int size) {
+        final RecordSizeCountStat val = map.get(key);
+        final RecordSizeCountStat recordStat = val == null ? new RecordSizeCountStat() : val;
+        recordStat.occurrence(size);
+        map.put(key, recordStat);
+    }
+
+    /**
+     * Handles WAL record.
+     *
+     * @param record record to handle.
+     * @param walPointer pointer, used to extract segment index.
+     * @param workDir true for work, false for archive folder.
+     */
+    void registerRecord(WALRecord record, WALPointer walPointer, boolean workDir) {
+        WALRecord.RecordType type = record.type();
+
+        if (type == WALRecord.RecordType.PAGE_RECORD)
+            registerPageSnapshot((PageSnapshot)record);
+        else if (type == WALRecord.RecordType.DATA_RECORD)
+            registerDataRecord((DataRecord)record);
+        else if (type == WALRecord.RecordType.TX_RECORD)
+            registerTxRecord((TxRecord)record);
+
+        incrementStat(type.toString(), record, recTypeSizes);
+
+        if (walPointer instanceof FileWALPointer) {
+            final FileWALPointer fPtr = (FileWALPointer)walPointer;
+
+            incrementStat(Long.toString(fPtr.index()), record, segmentsIndexes);
+            incrementStat(workDir ? "work" : "archive", record, segmentsFolder);
+        }
+    }
+
+    /**
+     * @param txRecord TX record to handle.
+     */
+    private void registerTxRecord(TxRecord txRecord) {
+        final TransactionState state = txRecord.state();
+
+        incrementStat(state.toString(), txRecord, txRecordAct);
+
+        int totalNodes = 0;
+        final Map<Short, Collection<Short>> map = txRecord.participatingNodes();
+
+        if (map != null) {
+            incrementStat(map.size(), txRecord, txRecordPrimNodesCnt);
+
+            final HashSet<Object> set = new HashSet<>(150);
+
+            for (Map.Entry<Short, Collection<Short>> next : map.entrySet()) {
+                set.add(next.getKey());
+                set.addAll(next.getValue());
+            }
+
+            totalNodes = set.size();
+
+            incrementStat(totalNodes, txRecord, txRecordNodesCnt);
+        }
+
+        final GridCacheVersion ver = txRecord.nearXidVersion();
+        if (ver != null) {
+            switch (state) {
+                case PREPARING:
+                case PREPARED:
+                    txStat.onTxPrepareStart(ver, map != null ? map.size() : 0, totalNodes);
+                    break;
+                case COMMITTED:
+                    txStat.onTxEnd(ver, true);
+                    break;
+                default:
+                    txStat.onTxEnd(ver, false);
+            }
+        }
+    }
+
+    /**
+     * @param record page snapshot record to handle.
+     */
+    private void registerPageSnapshot(PageSnapshot record) {
+        FullPageId fullPageId = record.fullPageId();
+        long pageId = fullPageId.pageId();
+
+        incrementStat(getPageType(record), record, pageSnapshotTypes);
+
+        final int idx = PageIdUtils.pageIndex(pageId);
+        final String idxAsStr = idx <= 100 ? Integer.toString(idx) : ">100";
+
+        incrementStat(idxAsStr, record, pageSnapshotIndexes);
+        incrementStat(fullPageId.groupId(), record, pageSnapshotCacheGrp);
+        incrementStat(PageIdUtils.partId(pageId), record, pageSnapshotPartId);
+    }
+
+    /**
+     * @param record data record to handle.
+     */
+    private void registerDataRecord(DataRecord record) {
+        final List<DataEntry> dataEntries = record.writeEntries();
+        if (!dataEntries.isEmpty()) {
+            boolean underTx = false;
+            for (DataEntry next : dataEntries) {
+                final int size = dataEntries.size() > 1 ? -1 : record.size();
+                incrementStat(next.op().toString(), dataEntryOperation, size);
+
+                incrementStat(next.cacheId(), dataEntryCacheId, size);
+
+                txStat.onDataEntry(next);
+
+                underTx |= next.nearXidVersion() != null;
+            }
+
+            incrementStat(underTx, record, dataRecordUnderTx);
+        }
+
+        incrementStat(dataEntries.size(), record, dataRecordEntriesCnt);
+    }
+
+    /**
+     * @param record page snapshot record.
+     * @return string identifier of page (IO) type.
+     */
+    private static String getPageType(PageSnapshot record) {
+        byte[] pageData = record.pageData();
+        ByteBuffer buf = ByteBuffer.allocateDirect(pageData.length);
+
+        try {
+            buf.order(ByteOrder.nativeOrder());
+            buf.put(pageData);
+
+            long addr = GridUnsafe.bufferAddress(buf);
+
+            int type = PageIO.getType(addr);
+            int ver = PageIO.getVersion(addr);
+
+            return PageIO.getPageIO(type, ver).getClass().getSimpleName();
+        }
+        catch (IgniteCheckedException ignored) {
+        }
+        finally {
+            ((DirectBuffer)buf).cleaner().clean();
+        }
+        return "";
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        final StringBuilder sb = new StringBuilder();
+
+        printSizeCountMap(sb, "WAL Segments: Source folder", segmentsFolder);
+        printSizeCountMap(sb, "WAL Segments: File index", segmentsIndexes);
+
+        printSizeCountMap(sb, "Record type", recTypeSizes);
+
+        printSizeCountMap(sb, "Tx Record: Action", txRecordAct);
+        printSizeCountMap(sb, "Tx Record: Primary nodes count", txRecordPrimNodesCnt);
+        printSizeCountMap(sb, "Tx Record: Nodes count", txRecordNodesCnt);
+
+        printSizeCountMap(sb, "Data Record: Entries count", dataRecordEntriesCnt);
+        printSizeCountMap(sb, "Data Record: Under TX", dataRecordUnderTx);
+
+        printSizeCountMap(sb, "Data Entry: Operations", dataEntryOperation);
+        printSizeCountMap(sb, "Data Entry: Cache ID", dataEntryCacheId);
+
+        printSizeCountMap(sb, "Page Snapshot: Page Types", pageSnapshotTypes);
+        printSizeCountMap(sb, "Page Snapshot: Indexes", pageSnapshotIndexes);
+        printSizeCountMap(sb, "Page Snapshot: Cache Groups", pageSnapshotCacheGrp);
+        printSizeCountMap(sb, "Page Snapshot: Partition ID", pageSnapshotPartId);
+
+        sb.append(txStat.toString());
+
+        return sb.toString();
+    }
+
+    /**
+     * @param sb buffer.
+     * @param mapName display name of map.
+     * @param map data.
+     */
+    private void printSizeCountMap(StringBuilder sb, String mapName, Map<?, RecordSizeCountStat> map) {
+        sb.append(mapName).append(": \n");
+        sb.append("key\tsize\tcount\tavg.size");
+        sb.append("\n");
+
+        final List<? extends Map.Entry<?, RecordSizeCountStat>> entries = new ArrayList<>(map.entrySet());
+
+        Collections.sort(entries, new Comparator<Map.Entry<?, RecordSizeCountStat>>() {
+            @Override public int compare(Map.Entry<?, RecordSizeCountStat> o1, Map.Entry<?, RecordSizeCountStat> o2) {
+                return -Integer.compare(o1.getValue().getCount(), o2.getValue().getCount());
+            }
+        });
+
+        RecordSizeCountStat others = new RecordSizeCountStat();
+        int otherRecords = 0;
+        int cnt = 0;
+
+        for (Map.Entry<?, RecordSizeCountStat> next : entries) {
+            if (cnt < DISPLAY_MAX) {
+                sb.append(next.getKey()).append("\t").append(next.getValue()).append("\t");
+                sb.append("\n");
+            }
+            else {
+                otherRecords++;
+                others.add(next.getValue());
+            }
+            cnt++;
+        }
+
+        if (otherRecords > 0) {
+            sb.append("... other ").append(otherRecords).append(" values").append("\t").append(others).append("\t");
+            sb.append("\n");
+        }
+
+        sb.append("\n");
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
index c803a36..c4a8c08 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
@@ -458,8 +458,6 @@ public class GridH2Table extends TableBase {
                     replaced = prevRow0 != null;
                 }
 
-                assert (replaced && prevRow0 != null) || (!replaced && prevRow0 == null) : "Replaced: " + replaced;
-
                 if (!replaced)
                     size.increment();
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryObjectFieldsQuerySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryObjectFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryObjectFieldsQuerySelfTest.java
index 6c34373..62515c3 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryObjectFieldsQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryObjectFieldsQuerySelfTest.java
@@ -103,11 +103,16 @@ public class IgniteBinaryObjectFieldsQuerySelfTest extends GridCommonAbstractTes
 
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
-        extClassLoader = getExternalClassLoader();
+        initExtClassLoader();
 
         startGrids(GRID_CNT);
     }
 
+    /** */
+    protected void initExtClassLoader() {
+        extClassLoader = getExternalClassLoader();
+    }
+
     /** {@inheritDoc} */
     @Override protected void afterTestsStopped() throws Exception {
         stopAllGrids();

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAbstractTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAbstractTest.java
index 83f5015..9af75ed 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAbstractTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAbstractTest.java
@@ -198,7 +198,7 @@ public class IgniteCacheLockPartitionOnAffinityRunAbstractTest extends GridCache
     /**
      * @throws Exception If failed.
      */
-    private void fillCaches() throws Exception {
+    protected void fillCaches() throws Exception {
         grid(0).createCache(Organization.class.getSimpleName());
         grid(0).createCache(Person.class.getSimpleName());
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest.java
index 71e737f..a4f398f 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest.java
@@ -94,7 +94,7 @@ public class IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest extends Igni
      * @param mode Atomicity mode.
      * @throws Exception If failed.
      */
-    private void createCache(String cacheName, CacheAtomicityMode mode) throws Exception {
+    protected void createCache(String cacheName, CacheAtomicityMode mode) throws Exception {
         CacheConfiguration ccfg = cacheConfiguration(grid(0).name());
         ccfg.setName(cacheName);
 
@@ -109,17 +109,27 @@ public class IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest extends Igni
     @Override protected void beforeTest() throws Exception {
         super.beforeTest();
 
+        createCaches();
+
+        awaitPartitionMapExchange();
+    }
+
+    /** */
+    protected void createCaches() throws Exception {
         key.set(0);
         createCache(ATOMIC_CACHE, CacheAtomicityMode.ATOMIC);
         createCache(TRANSACT_CACHE, CacheAtomicityMode.TRANSACTIONAL);
+    }
 
-        awaitPartitionMapExchange();
+    /** */
+    protected void destroyCaches() throws Exception {
+        grid(0).destroyCache(ATOMIC_CACHE);
+        grid(0).destroyCache(TRANSACT_CACHE);
     }
 
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
-        grid(0).destroyCache(ATOMIC_CACHE);
-        grid(0).destroyCache(TRANSACT_CACHE);
+        destroyCaches();
 
         super.afterTest();
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java
index 0c644e3..fc1cea6 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java
@@ -105,7 +105,6 @@ public class IgniteCacheQueryNodeRestartSelfTest extends GridCacheAbstractSelfTe
     public void testRestarts() throws Exception {
         int duration = 60 * 1000;
         int qryThreadNum = 10;
-        final long nodeLifeTime = 2 * 1000;
         final int logFreq = 50;
 
         final IgniteCache<Integer, Integer> cache = grid(0).cache(DEFAULT_CACHE_NAME);
@@ -158,27 +157,7 @@ public class IgniteCacheQueryNodeRestartSelfTest extends GridCacheAbstractSelfTe
         for (int i = 0; i < GRID_CNT; i++)
             grid(i).events().localListen(lsnr, EventType.EVT_CACHE_REBALANCE_STOPPED);
 
-        IgniteInternalFuture<?> fut2 = multithreadedAsync(new Callable<Object>() {
-            @SuppressWarnings({"BusyWait"})
-            @Override public Object call() throws Exception {
-                while (!done.get()) {
-                    int idx = GRID_CNT;
-
-                    startGrid(idx);
-
-                    Thread.sleep(nodeLifeTime);
-
-                    stopGrid(idx);
-
-                    int c = restartCnt.incrementAndGet();
-
-                    if (c % logFreq == 0)
-                        info("Node restarts: " + c);
-                }
-
-                return true;
-            }
-        }, 1, "restart-thread");
+        IgniteInternalFuture<?> fut2 = createRestartAction(done, restartCnt);
 
         Thread.sleep(duration);
 
@@ -204,6 +183,39 @@ public class IgniteCacheQueryNodeRestartSelfTest extends GridCacheAbstractSelfTe
         assert success;
     }
 
+    /**
+     *
+     */
+    protected IgniteInternalFuture createRestartAction(final AtomicBoolean done, final AtomicInteger restartCnt) throws Exception {
+        return multithreadedAsync(new Callable<Object>() {
+            /** */
+            private final long nodeLifeTime = 2 * 1000;
+
+            /** */
+            private final int logFreq = 50;
+
+            @SuppressWarnings({"BusyWait"})
+            @Override public Object call() throws Exception {
+                while (!done.get()) {
+                    int idx = GRID_CNT;
+
+                    startGrid(idx);
+
+                    Thread.sleep(nodeLifeTime);
+
+                    stopGrid(idx);
+
+                    int c = restartCnt.incrementAndGet();
+
+                    if (c % logFreq == 0)
+                        info("Node restarts: " + c);
+                }
+
+                return true;
+            }
+        }, 1, "restart-thread");
+    }
+
     /** Listener that will wait for specified number of events received. */
     private class CollectingEventListener implements IgnitePredicate<Event> {
         /** Registered events count. */

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest.java
new file mode 100644
index 0000000..c7663ea
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.database.baseline;
+
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest extends IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setInitialSize(200 * 1024 * 1024)
+                        .setMaxSize(200 * 1024 * 1024)
+                        .setPersistenceEnabled(true)
+                )
+        );
+
+        return cfg;
+    }
+
+    @Override protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception {
+        CacheConfiguration ccfg = super.cacheConfiguration(igniteInstanceName);
+
+        ccfg.setBackups(2);
+
+        return ccfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        int gridCnt = gridCount();
+
+        startGrids(gridCnt + 1);
+
+        grid(0).active(true);
+
+        stopGrid(gridCnt);
+
+        startGrid(gridCnt + 1);
+
+        fillCaches();
+
+        createCaches();
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        destroyCaches();
+
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunTxCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunTxCacheTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunTxCacheTest.java
new file mode 100644
index 0000000..8812829
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunTxCacheTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.database.baseline;
+
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.IgniteCacheLockPartitionOnAffinityRunTxCacheOpTest;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteBaselineLockPartitionOnAffinityRunTxCacheTest extends IgniteCacheLockPartitionOnAffinityRunTxCacheOpTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setInitialSize(200 * 1024 * 1024)
+                        .setMaxSize(200 * 1024 * 1024)
+                        .setPersistenceEnabled(true)
+                )
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        int gridCnt = gridCount();
+
+        startGrids(gridCnt + 1);
+
+        grid(0).active(true);
+
+        stopGrid(gridCnt);
+
+        startGrid(gridCnt + 1);
+
+        fillCaches();
+
+        createCaches();
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        destroyCaches();
+
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java
new file mode 100644
index 0000000..357d8e6
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.database.baseline;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheQueryNodeRestartSelfTest;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteChangingBaselineCacheQueryNodeRestartSelfTest extends IgniteCacheQueryNodeRestartSelfTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setMaxSize(200 * 1024 * 1024)
+                        .setPersistenceEnabled(true)
+                )
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(gridCount());
+
+        initStoreStrategy();
+
+        grid(0).active(true);
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteInternalFuture createRestartAction(final AtomicBoolean done, final AtomicInteger restartCnt) throws Exception {
+        return multithreadedAsync(new Callable<Object>() {
+            /** */
+            private final long baselineTopChangeInterval = 30 * 1000;
+
+            /** */
+            private final int logFreq = 50;
+
+            /** flag to indicate that last operation was changing BaselineTopology up (add node) */
+            private boolean lastOpChangeUp;
+
+            @SuppressWarnings({"BusyWait"})
+            @Override public Object call() throws Exception {
+                while (!done.get()) {
+                    if (lastOpChangeUp) {
+                        //need to do change down: stop node, set new BLT without it
+                        stopGrid(gridCount());
+
+                        lastOpChangeUp = false;
+                    }
+                    else {
+                        startGrid(gridCount());
+
+                        lastOpChangeUp = true;
+                    }
+
+                    grid(0).cluster().setBaselineTopology(baselineNodes(grid(0).cluster().forServers().nodes()));
+
+                    Thread.sleep(baselineTopChangeInterval);
+
+                    int c = restartCnt.incrementAndGet();
+
+                    if (c % logFreq == 0)
+                        info("BaselineTopology changes: " + c);
+                }
+
+                return true;
+            }
+        }, 1, "restart-thread");
+    }
+
+    /** */
+    private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+        Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+        for (ClusterNode clN : clNodes)
+            res.add(clN);
+
+        return res;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineBinObjFieldsQuerySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineBinObjFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineBinObjFieldsQuerySelfTest.java
new file mode 100644
index 0000000..e9d1b79
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineBinObjFieldsQuerySelfTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.database.baseline;
+
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.IgniteBinaryObjectFieldsQuerySelfTest;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteStableBaselineBinObjFieldsQuerySelfTest extends IgniteBinaryObjectFieldsQuerySelfTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setMaxSize(200 * 1024 * 1024)
+                        .setPersistenceEnabled(true)
+                )
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        initExtClassLoader();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(GRID_CNT + 1);
+
+        grid(0).active(true);
+
+        stopGrid(GRID_CNT);
+
+        startGrid(GRID_CNT + 1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        GridTestUtils.deleteDbFiles();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineCacheQueryNodeRestartsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineCacheQueryNodeRestartsSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineCacheQueryNodeRestartsSelfTest.java
new file mode 100644
index 0000000..849bbb6
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineCacheQueryNodeRestartsSelfTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.database.baseline;
+
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheQueryNodeRestartSelfTest;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteStableBaselineCacheQueryNodeRestartsSelfTest extends IgniteCacheQueryNodeRestartSelfTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setMaxSize(200 * 1024 * 1024)
+                        .setPersistenceEnabled(true)
+                )
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(gridCount() + 1);
+
+        initStoreStrategy();
+
+        grid(0).active(true);
+
+        stopGrid(gridCount());
+
+        startGrid(gridCount() + 1);
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheAffinityRunTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheAffinityRunTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheAffinityRunTestSuite.java
index ef00fc3..e9c7b79 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheAffinityRunTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheAffinityRunTestSuite.java
@@ -22,6 +22,8 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheLockPartitionOnAff
 import org.apache.ignite.internal.processors.cache.IgniteCacheLockPartitionOnAffinityRunTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheLockPartitionOnAffinityRunTxCacheOpTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheLockPartitionOnAffinityRunWithCollisionSpiTest;
+import org.apache.ignite.internal.processors.database.baseline.IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest;
+import org.apache.ignite.internal.processors.database.baseline.IgniteBaselineLockPartitionOnAffinityRunTxCacheTest;
 
 /**
  * Compute and Cache tests for affinityRun/Call. These tests is extracted into separate suite
@@ -38,6 +40,8 @@ public class IgniteCacheAffinityRunTestSuite extends TestSuite {
         suite.addTestSuite(IgniteCacheLockPartitionOnAffinityRunTest.class);
         suite.addTestSuite(IgniteCacheLockPartitionOnAffinityRunWithCollisionSpiTest.class);
         suite.addTestSuite(IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest.class);
+        suite.addTestSuite(IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest.class);
+        suite.addTestSuite(IgniteBaselineLockPartitionOnAffinityRunTxCacheTest.class);
         suite.addTestSuite(IgniteCacheLockPartitionOnAffinityRunTxCacheOpTest.class);
 
         return suite;

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index cf5c9a6..8cfa14e 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -136,6 +136,7 @@ import org.apache.ignite.internal.processors.cache.query.IgniteCacheQueryCacheDe
 import org.apache.ignite.internal.processors.cache.query.IndexingSpiQuerySelfTest;
 import org.apache.ignite.internal.processors.cache.query.IndexingSpiQueryTxSelfTest;
 import org.apache.ignite.internal.processors.client.ClientConnectorConfigurationValidationSelfTest;
+import org.apache.ignite.internal.processors.database.baseline.IgniteStableBaselineBinObjFieldsQuerySelfTest;
 import org.apache.ignite.internal.processors.query.IgniteSqlDefaultValueTest;
 import org.apache.ignite.internal.processors.query.IgniteSqlDistributedJoinSelfTest;
 import org.apache.ignite.internal.processors.query.IgniteSqlSkipReducerOnUpdateDmlFlagSelfTest;
@@ -248,6 +249,7 @@ public class IgniteCacheQuerySelfTestSuite extends TestSuite {
         suite.addTestSuite(GridCacheCrossCacheQuerySelfTest.class);
         suite.addTestSuite(GridCacheQuerySerializationSelfTest.class);
         suite.addTestSuite(IgniteBinaryObjectFieldsQuerySelfTest.class);
+        suite.addTestSuite(IgniteStableBaselineBinObjFieldsQuerySelfTest.class);
         suite.addTestSuite(IgniteBinaryWrappedObjectFieldsQuerySelfTest.class);
         suite.addTestSuite(IgniteCacheQueryH2IndexingLeakTest.class);
         suite.addTestSuite(IgniteCacheQueryNoRebalanceSelfTest.class);

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
index abe06ec..11d98e2 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
@@ -42,6 +42,8 @@ import org.apache.ignite.internal.processors.cache.index.DynamicIndexPartitioned
 import org.apache.ignite.internal.processors.cache.index.DynamicIndexReplicatedAtomicConcurrentSelfTest;
 import org.apache.ignite.internal.processors.cache.index.DynamicIndexReplicatedTransactionalConcurrentSelfTest;
 import org.apache.ignite.internal.processors.cache.query.ScanQueryOffheapExpiryPolicySelfTest;
+import org.apache.ignite.internal.processors.database.baseline.IgniteChangingBaselineCacheQueryNodeRestartSelfTest;
+import org.apache.ignite.internal.processors.database.baseline.IgniteStableBaselineCacheQueryNodeRestartsSelfTest;
 import org.apache.ignite.internal.processors.query.IgniteCacheGroupsCompareQueryTest;
 import org.apache.ignite.internal.processors.query.IgniteCacheGroupsSqlDistributedJoinSelfTest;
 import org.apache.ignite.internal.processors.query.IgniteCacheGroupsSqlSegmentedIndexMultiNodeSelfTest;
@@ -85,6 +87,8 @@ public class IgniteCacheQuerySelfTestSuite2 extends TestSuite {
         suite.addTestSuite(IgniteCacheClientQueryReplicatedNodeRestartSelfTest.class);
         suite.addTestSuite(IgniteCacheQueryNodeFailTest.class);
         suite.addTestSuite(IgniteCacheQueryNodeRestartSelfTest.class);
+        suite.addTestSuite(IgniteChangingBaselineCacheQueryNodeRestartSelfTest.class);
+        suite.addTestSuite(IgniteStableBaselineCacheQueryNodeRestartsSelfTest.class);
         suite.addTestSuite(IgniteCacheQueryNodeRestartSelfTest2.class);
         suite.addTestSuite(IgniteCacheQueryNodeRestartTxSelfTest.class);
         suite.addTestSuite(IgniteCacheSqlQueryMultiThreadedSelfTest.class);

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterParityTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterParityTest.cs
index b940d09..4b4279f 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterParityTest.cs
@@ -38,7 +38,11 @@
         /** Members that are missing on .NET side and should be added in future. */
         private static readonly string[] MissingMembers =
         {
-            "enableStatistics"  // IGNITE-7276
+            "enableStatistics",  // IGNITE-7276
+            
+            // IGNITE-7301
+            "active",
+            "setBaselineTopology"
         };
 
         /// <summary>

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
index 235b177..410d911 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
@@ -78,7 +78,8 @@ namespace Apache.Ignite.Core.Tests.ApiParity
             "isCacheSanityCheckEnabled",
             "TimeServerPortBase",
             "TimeServerPortRange",
-            "IncludeProperties"
+            "IncludeProperties",
+            "isAutoActivationEnabled"  // IGNITE-7301
         };
 
         /// <summary>


[10/17] ignite git commit: IEP-4 Baseline topology for persistent caches (Phase 1) Contributed by: Dmitriy Govorukhin Dmitry Pavlov Eduard Shangareev Ily

Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java
index 9ed4000..c2cc171 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java
@@ -17,12 +17,16 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.partstate;
 
+import java.util.HashSet;
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Set;
 import java.util.TreeMap;
 import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -32,8 +36,13 @@ import org.jetbrains.annotations.Nullable;
  */
 public class PartitionAllocationMap {
     /** Maps following pairs: (groupId, partId) -> (lastAllocatedCount, allocatedCount) */
+    @GridToStringInclude
     private final NavigableMap<GroupPartitionId, PagesAllocationRange> map = new TreeMap<>();
 
+    /** Partitions forced to be skipped. */
+    @GridToStringInclude
+    private final Set<GroupPartitionId> skippedParts = new HashSet<>();
+
     /**
      * Returns the value to which the specified key is mapped,
      * or {@code null} if this map contains no mapping for the key.
@@ -46,6 +55,13 @@ public class PartitionAllocationMap {
     }
 
     /**
+     * @param fullPageId Full page id.
+     */
+    @Nullable public PagesAllocationRange get(FullPageId fullPageId) {
+        return map.get(createCachePartId(fullPageId));
+    }
+
+    /**
      * Extracts partition information from full page ID
      *
      * @param fullId page related to some cache
@@ -81,6 +97,20 @@ public class PartitionAllocationMap {
     }
 
     /**
+     * Forces the index partition for the given group ID to be skipped in collected map.
+     *
+     * @param grpId Group ID to skip.
+     * @return {@code true} if skipped partition was added to the ignore list during this call.
+     */
+    public boolean forceSkipIndexPartition(int grpId) {
+        GroupPartitionId idxId = new GroupPartitionId(grpId, PageIdAllocator.INDEX_PARTITION);
+
+        map.remove(idxId);
+
+        return skippedParts.add(idxId);
+    }
+
+    /**
      * Returns next (higher) key for provided cache and partition or null
      *
      * @param key cache and partition to search
@@ -107,7 +137,11 @@ public class PartitionAllocationMap {
      * <tt>key</tt>.
      */
     public PagesAllocationRange put(GroupPartitionId key, PagesAllocationRange val) {
-        return map.put(key, val);
+        return !skippedParts.contains(key) ? map.put(key, val) : null;
     }
 
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(PartitionAllocationMap.class, this);
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java
index 45c0b11..c23adda 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java
@@ -82,6 +82,13 @@ public class IgniteCacheSnapshotManager<T extends SnapshotOperation> extends Gri
     /**
      *
      */
+    public boolean partitionsAreFrozen(CacheGroupContext grp) {
+        return false;
+    }
+
+    /**
+     *
+     */
     public void restoreState() throws IgniteCheckedException {
         // No-op.
     }
@@ -170,4 +177,11 @@ public class IgniteCacheSnapshotManager<T extends SnapshotOperation> extends Gri
     @Override public void onDeActivate(GridKernalContext kctx) {
         // No-op.
     }
+
+    /**
+     * @return {@code True} if TX READ records must be logged in WAL.
+     */
+    public boolean needTxReadLogging() {
+        return false;
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDiscoveryMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDiscoveryMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDiscoveryMessage.java
index d88d96e..bcad7df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDiscoveryMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDiscoveryMessage.java
@@ -29,5 +29,10 @@ public interface SnapshotDiscoveryMessage extends DiscoveryCustomMessage {
      *
      * @return True if exchange is needed, false in other case.
      */
-    boolean needExchange();
+    public boolean needExchange();
+
+    /**
+     *
+     */
+    public boolean needAssignPartitions();
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
index cb6898a..6b87d32 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
@@ -3706,6 +3706,7 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements
          * @param pageAddr Page address.
          * @param walPlc Full page WAL record policy.
          * @param io IO.
+         *
          * @param cnt Count.
          * @param idx Index to remove.
          * @throws IgniteCheckedException If failed.

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java
new file mode 100644
index 0000000..ca63f27
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java
@@ -0,0 +1,1256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.processors.cache.persistence.Storable;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.util.GridStringBuilder;
+import org.apache.ignite.internal.util.typedef.internal.SB;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Data pages IO.
+ */
+public abstract class AbstractDataPageIO<T extends Storable> extends PageIO {
+
+    /** */
+    private static final int SHOW_ITEM = 0b0001;
+
+    /** */
+    private static final int SHOW_PAYLOAD_LEN = 0b0010;
+
+    /** */
+    private static final int SHOW_LINK = 0b0100;
+
+    /** */
+    private static final int FREE_LIST_PAGE_ID_OFF = COMMON_HEADER_END;
+
+    /** */
+    private static final int FREE_SPACE_OFF = FREE_LIST_PAGE_ID_OFF + 8;
+
+    /** */
+    private static final int DIRECT_CNT_OFF = FREE_SPACE_OFF + 2;
+
+    /** */
+    private static final int INDIRECT_CNT_OFF = DIRECT_CNT_OFF + 1;
+
+    /** */
+    private static final int FIRST_ENTRY_OFF = INDIRECT_CNT_OFF + 1;
+
+    /** */
+    public static final int ITEMS_OFF = FIRST_ENTRY_OFF + 2;
+
+    /** */
+    private static final int ITEM_SIZE = 2;
+
+    /** */
+    private static final int PAYLOAD_LEN_SIZE = 2;
+
+    /** */
+    private static final int LINK_SIZE = 8;
+
+    /** */
+    private static final int FRAGMENTED_FLAG = 0b10000000_00000000;
+
+    /** */
+    public static final int MIN_DATA_PAGE_OVERHEAD = ITEMS_OFF + ITEM_SIZE + PAYLOAD_LEN_SIZE + LINK_SIZE;
+
+    /**
+     * @param ver Page format version.
+     */
+    protected AbstractDataPageIO(int type, int ver) {
+        super(type, ver);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
+        super.initNewPage(pageAddr, pageId, pageSize);
+
+        setEmptyPage(pageAddr, pageSize);
+        setFreeListPageId(pageAddr, 0L);
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param pageSize Page size.
+     */
+    private void setEmptyPage(long pageAddr, int pageSize) {
+        setDirectCount(pageAddr, 0);
+        setIndirectCount(pageAddr, 0);
+        setFirstEntryOffset(pageAddr, pageSize, pageSize);
+        setRealFreeSpace(pageAddr, pageSize - ITEMS_OFF, pageSize);
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param freeListPageId Free list page ID.
+     */
+    public void setFreeListPageId(long pageAddr, long freeListPageId) {
+        PageUtils.putLong(pageAddr, FREE_LIST_PAGE_ID_OFF, freeListPageId);
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @return Free list page ID.
+     */
+    public long getFreeListPageId(long pageAddr) {
+        return PageUtils.getLong(pageAddr, FREE_LIST_PAGE_ID_OFF);
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param dataOff Data offset.
+     * @param show What elements of data page entry to show in the result.
+     * @return Data page entry size.
+     */
+    private int getPageEntrySize(long pageAddr, int dataOff, int show) {
+        int payloadLen = PageUtils.getShort(pageAddr, dataOff) & 0xFFFF;
+
+        if ((payloadLen & FRAGMENTED_FLAG) != 0)
+            payloadLen &= ~FRAGMENTED_FLAG; // We are fragmented and have a link.
+        else
+            show &= ~SHOW_LINK; // We are not fragmented, never have a link.
+
+        return getPageEntrySize(payloadLen, show);
+    }
+
+    /**
+     * @param payloadLen Length of the payload, may be a full data row or a row fragment length.
+     * @param show What elements of data page entry to show in the result.
+     * @return Data page entry size.
+     */
+    private int getPageEntrySize(int payloadLen, int show) {
+        assert payloadLen > 0 : payloadLen;
+
+        int res = payloadLen;
+
+        if ((show & SHOW_LINK) != 0)
+            res += LINK_SIZE;
+
+        if ((show & SHOW_ITEM) != 0)
+            res += ITEM_SIZE;
+
+        if ((show & SHOW_PAYLOAD_LEN) != 0)
+            res += PAYLOAD_LEN_SIZE;
+
+        return res;
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param dataOff Entry data offset.
+     * @param pageSize Page size.
+     */
+    private void setFirstEntryOffset(long pageAddr, int dataOff, int pageSize) {
+        assert dataOff >= ITEMS_OFF + ITEM_SIZE && dataOff <= pageSize : dataOff;
+
+        PageUtils.putShort(pageAddr, FIRST_ENTRY_OFF, (short)dataOff);
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @return Entry data offset.
+     */
+    private int getFirstEntryOffset(long pageAddr) {
+        return PageUtils.getShort(pageAddr, FIRST_ENTRY_OFF) & 0xFFFF;
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param freeSpace Free space.
+     * @param pageSize Page size.
+     */
+    private void setRealFreeSpace(long pageAddr, int freeSpace, int pageSize) {
+        assert freeSpace == actualFreeSpace(pageAddr, pageSize) : freeSpace + " != " + actualFreeSpace(pageAddr, pageSize);
+
+        PageUtils.putShort(pageAddr, FREE_SPACE_OFF, (short)freeSpace);
+    }
+
+    /**
+     * Free space refers to a "max row size (without any data page specific overhead) which is guaranteed to fit into
+     * this data page".
+     *
+     * @param pageAddr Page address.
+     * @return Free space.
+     */
+    public int getFreeSpace(long pageAddr) {
+        if (getFreeItemSlots(pageAddr) == 0)
+            return 0;
+
+        int freeSpace = getRealFreeSpace(pageAddr);
+
+        // We reserve size here because of getFreeSpace() method semantics (see method javadoc).
+        // It means that we must be able to accommodate a row of size which is equal to getFreeSpace(),
+        // plus we will have data page overhead: header of the page as well as item, payload length and
+        // possibly a link to the next row fragment.
+        freeSpace -= ITEM_SIZE + PAYLOAD_LEN_SIZE + LINK_SIZE;
+
+        return freeSpace < 0 ? 0 : freeSpace;
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @return {@code true} If there is no useful data in this page.
+     */
+    public boolean isEmpty(long pageAddr) {
+        return getDirectCount(pageAddr) == 0;
+    }
+
+    /**
+     * Equivalent for {@link #actualFreeSpace(long, int)} but reads saved value.
+     *
+     * @param pageAddr Page address.
+     * @return Free space.
+     */
+    private int getRealFreeSpace(long pageAddr) {
+        return PageUtils.getShort(pageAddr, FREE_SPACE_OFF);
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param cnt Direct count.
+     */
+    private void setDirectCount(long pageAddr, int cnt) {
+        assert checkCount(cnt) : cnt;
+
+        PageUtils.putByte(pageAddr, DIRECT_CNT_OFF, (byte)cnt);
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @return Direct count.
+     */
+    private int getDirectCount(long pageAddr) {
+        return PageUtils.getByte(pageAddr, DIRECT_CNT_OFF) & 0xFF;
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param c Closure.
+     * @param <T> Closure return type.
+     * @return Collection of closure results for all items in page.
+     * @throws IgniteCheckedException In case of error in closure body.
+     */
+    public <T> List<T> forAllItems(long pageAddr, CC<T> c) throws IgniteCheckedException {
+        long pageId = getPageId(pageAddr);
+
+        int cnt = getDirectCount(pageAddr);
+
+        List<T> res = new ArrayList<>(cnt);
+
+        for (int i = 0; i < cnt; i++) {
+            long link = PageIdUtils.link(pageId, i);
+
+            res.add(c.apply(link));
+        }
+
+        return res;
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param cnt Indirect count.
+     */
+    private void setIndirectCount(long pageAddr, int cnt) {
+        assert checkCount(cnt) : cnt;
+
+        PageUtils.putByte(pageAddr, INDIRECT_CNT_OFF, (byte)cnt);
+    }
+
+    /**
+     * @param idx Index.
+     * @return {@code true} If the index is valid.
+     */
+    protected boolean checkIndex(int idx) {
+        return idx >= 0 && idx < 0xFF;
+    }
+
+    /**
+     * @param cnt Counter value.
+     * @return {@code true} If the counter fits 1 byte.
+     */
+    private boolean checkCount(int cnt) {
+        return cnt >= 0 && cnt <= 0xFF;
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @return Indirect count.
+     */
+    private int getIndirectCount(long pageAddr) {
+        return PageUtils.getByte(pageAddr, INDIRECT_CNT_OFF) & 0xFF;
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @return Number of free entry slots.
+     */
+    private int getFreeItemSlots(long pageAddr) {
+        return 0xFF - getDirectCount(pageAddr);
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
+     * @param directCnt Direct items count.
+     * @param indirectCnt Indirect items count.
+     * @return Found index of indirect item.
+     */
+    private int findIndirectItemIndex(long pageAddr, int itemId, int directCnt, int indirectCnt) {
+        int low = directCnt;
+        int high = directCnt + indirectCnt - 1;
+
+        while (low <= high) {
+            int mid = (low + high) >>> 1;
+
+            int cmp = Integer.compare(itemId(getItem(pageAddr, mid)), itemId);
+
+            if (cmp < 0)
+                low = mid + 1;
+            else if (cmp > 0)
+                high = mid - 1;
+            else
+                return mid; // found
+        }
+
+        throw new IllegalStateException("Item not found: " + itemId);
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param pageSize Page size.
+     * @return String representation.
+     */
+    private String printPageLayout(long pageAddr, int pageSize) {
+        SB b = new SB();
+
+        printPageLayout(pageAddr, pageSize, b);
+
+        return b.toString();
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param pageSize Page size.
+     * @param b B.
+     */
+    protected void printPageLayout(long pageAddr, int pageSize, GridStringBuilder b) {
+        int directCnt = getDirectCount(pageAddr);
+        int indirectCnt = getIndirectCount(pageAddr);
+        int free = getRealFreeSpace(pageAddr);
+
+        boolean valid = directCnt >= indirectCnt;
+
+        b.appendHex(PageIO.getPageId(pageAddr)).a(" [");
+
+        int entriesSize = 0;
+
+        for (int i = 0; i < directCnt; i++) {
+            if (i != 0)
+                b.a(", ");
+
+            short item = getItem(pageAddr, i);
+
+            if (item < ITEMS_OFF || item >= pageSize)
+                valid = false;
+
+            entriesSize += getPageEntrySize(pageAddr, item, SHOW_PAYLOAD_LEN | SHOW_LINK);
+
+            b.a(item);
+        }
+
+        b.a("][");
+
+        Collection<Integer> set = new HashSet<>();
+
+        for (int i = directCnt; i < directCnt + indirectCnt; i++) {
+            if (i != directCnt)
+                b.a(", ");
+
+            short item = getItem(pageAddr, i);
+
+            int itemId = itemId(item);
+            int directIdx = directItemIndex(item);
+
+            if (!set.add(directIdx) || !set.add(itemId))
+                valid = false;
+
+            assert indirectItem(itemId, directIdx) == item;
+
+            if (itemId < directCnt || directIdx < 0 || directIdx >= directCnt)
+                valid = false;
+
+            if (i > directCnt && itemId(getItem(pageAddr, i - 1)) >= itemId)
+                valid = false;
+
+
+            b.a(itemId).a('^').a(directIdx);
+        }
+
+        b.a("][free=").a(free);
+
+        int actualFree = pageSize - ITEMS_OFF - (entriesSize + (directCnt + indirectCnt) * ITEM_SIZE);
+
+        if (free != actualFree) {
+            b.a(", actualFree=").a(actualFree);
+
+            valid = false;
+        }
+        else
+            b.a("]");
+
+        assert valid : b.toString();
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
+     * @param pageSize Page size.
+     * @return Data entry offset in bytes.
+     */
+    protected int getDataOffset(long pageAddr, int itemId, int pageSize) {
+        assert checkIndex(itemId) : itemId;
+
+        int directCnt = getDirectCount(pageAddr);
+
+        assert directCnt > 0 : "itemId=" + itemId + ", directCnt=" + directCnt + ", page=" + printPageLayout(pageAddr, pageSize);
+
+        if (itemId >= directCnt) { // Need to do indirect lookup.
+            int indirectCnt = getIndirectCount(pageAddr);
+
+            // Must have indirect items here.
+            assert indirectCnt > 0 : "itemId=" + itemId + ", directCnt=" + directCnt + ", indirectCnt=" + indirectCnt +
+                ", page=" + printPageLayout(pageAddr, pageSize);
+
+            int indirectItemIdx = findIndirectItemIndex(pageAddr, itemId, directCnt, indirectCnt);
+
+            assert indirectItemIdx >= directCnt : indirectItemIdx + " " + directCnt;
+            assert indirectItemIdx < directCnt + indirectCnt : indirectItemIdx + " " + directCnt + " " + indirectCnt;
+
+            itemId = directItemIndex(getItem(pageAddr, indirectItemIdx));
+
+            assert itemId >= 0 && itemId < directCnt : itemId + " " + directCnt + " " + indirectCnt; // Direct item.
+        }
+
+        return directItemToOffset(getItem(pageAddr, itemId));
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param dataOff Points to the entry start.
+     * @return Link to the next entry fragment or 0 if no fragments left or if entry is not fragmented.
+     */
+    private long getNextFragmentLink(long pageAddr, int dataOff) {
+        assert isFragmented(pageAddr, dataOff);
+
+        return PageUtils.getLong(pageAddr, dataOff + PAYLOAD_LEN_SIZE);
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param dataOff Data offset.
+     * @return {@code true} If the data row is fragmented across multiple pages.
+     */
+    protected boolean isFragmented(long pageAddr, int dataOff) {
+        return (PageUtils.getShort(pageAddr, dataOff) & FRAGMENTED_FLAG) != 0;
+    }
+
+    /**
+     * Sets position to start of actual fragment data and limit to it's end.
+     *
+     * @param pageAddr Page address.
+     * @param itemId Item to position on.
+     * @param pageSize Page size.
+     * @return {@link DataPagePayload} object.
+     */
+    public DataPagePayload readPayload(final long pageAddr, final int itemId, final int pageSize) {
+        int dataOff = getDataOffset(pageAddr, itemId, pageSize);
+
+        boolean fragmented = isFragmented(pageAddr, dataOff);
+        long nextLink = fragmented ? getNextFragmentLink(pageAddr, dataOff) : 0;
+        int payloadSize = getPageEntrySize(pageAddr, dataOff, 0);
+
+        return new DataPagePayload(dataOff + PAYLOAD_LEN_SIZE + (fragmented ? LINK_SIZE : 0),
+            payloadSize,
+            nextLink);
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param idx Item index.
+     * @return Item.
+     */
+    private short getItem(long pageAddr, int idx) {
+        return PageUtils.getShort(pageAddr, itemOffset(idx));
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param idx Item index.
+     * @param item Item.
+     */
+    private void setItem(long pageAddr, int idx, short item) {
+        PageUtils.putShort(pageAddr, itemOffset(idx), item);
+    }
+
+    /**
+     * @param idx Index of the item.
+     * @return Offset in buffer.
+     */
+    private int itemOffset(int idx) {
+        assert checkIndex(idx) : idx;
+
+        return ITEMS_OFF + idx * ITEM_SIZE;
+    }
+
+    /**
+     * @param directItem Direct item.
+     * @return Offset of an entry payload inside of the page.
+     */
+    private int directItemToOffset(short directItem) {
+        return directItem & 0xFFFF;
+    }
+
+    /**
+     * @param dataOff Data offset.
+     * @return Direct item.
+     */
+    private short directItemFromOffset(int dataOff) {
+        assert dataOff >= ITEMS_OFF + ITEM_SIZE && dataOff < Short.MAX_VALUE : dataOff;
+
+        return (short)dataOff;
+    }
+
+    /**
+     * @param indirectItem Indirect item.
+     * @return Index of corresponding direct item.
+     */
+    private int directItemIndex(short indirectItem) {
+        return indirectItem & 0xFF;
+    }
+
+    /**
+     * @param indirectItem Indirect item.
+     * @return Fixed item ID (the index used for referencing an entry from the outside).
+     */
+    private int itemId(short indirectItem) {
+        return (indirectItem & 0xFFFF) >>> 8;
+    }
+
+    /**
+     * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
+     * @param directItemIdx Index of corresponding direct item.
+     * @return Indirect item.
+     */
+    private short indirectItem(int itemId, int directItemIdx) {
+        assert checkIndex(itemId) : itemId;
+        assert checkIndex(directItemIdx) : directItemIdx;
+
+        return (short)((itemId << 8) | directItemIdx);
+    }
+
+    /**
+     * Move the last direct item to the free slot and reference it with indirect item on the same place.
+     *
+     * @param pageAddr Page address.
+     * @param freeDirectIdx Free slot.
+     * @param directCnt Direct items count.
+     * @param indirectCnt Indirect items count.
+     * @return {@code true} If the last direct item already had corresponding indirect item.
+     */
+    private boolean moveLastItem(long pageAddr, int freeDirectIdx, int directCnt, int indirectCnt) {
+        int lastIndirectId = findIndirectIndexForLastDirect(pageAddr, directCnt, indirectCnt);
+
+        int lastItemId = directCnt - 1;
+
+        assert lastItemId != freeDirectIdx;
+
+        short indirectItem = indirectItem(lastItemId, freeDirectIdx);
+
+        assert itemId(indirectItem) == lastItemId && directItemIndex(indirectItem) == freeDirectIdx;
+
+        setItem(pageAddr, freeDirectIdx, getItem(pageAddr, lastItemId));
+        setItem(pageAddr, lastItemId, indirectItem);
+
+        assert getItem(pageAddr, lastItemId) == indirectItem;
+
+        if (lastIndirectId != -1) { // Fix pointer to direct item.
+            setItem(pageAddr, lastIndirectId, indirectItem(itemId(getItem(pageAddr, lastIndirectId)), freeDirectIdx));
+
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param directCnt Direct items count.
+     * @param indirectCnt Indirect items count.
+     * @return Index of indirect item for the last direct item.
+     */
+    private int findIndirectIndexForLastDirect(long pageAddr, int directCnt, int indirectCnt) {
+        int lastDirectId = directCnt - 1;
+
+        for (int i = directCnt, end = directCnt + indirectCnt; i < end; i++) {
+            short item = getItem(pageAddr, i);
+
+            if (directItemIndex(item) == lastDirectId)
+                return i;
+        }
+
+        return -1;
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param itemId Item ID.
+     * @param pageSize Page size.
+     * @param payload Row data.
+     * @param row Row.
+     * @param rowSize Row size.
+     * @return {@code True} if entry is not fragmented.
+     * @throws IgniteCheckedException If failed.
+     */
+    public boolean updateRow(
+        final long pageAddr,
+        int itemId,
+        int pageSize,
+        @Nullable byte[] payload,
+        @Nullable T row,
+        final int rowSize) throws IgniteCheckedException {
+        assert checkIndex(itemId) : itemId;
+        assert row != null ^ payload != null;
+
+        final int dataOff = getDataOffset(pageAddr, itemId, pageSize);
+
+        if (isFragmented(pageAddr, dataOff))
+            return false;
+
+        if (row != null)
+            writeRowData(pageAddr, dataOff, rowSize, row, false);
+        else
+            writeRowData(pageAddr, dataOff, payload);
+
+        return true;
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
+     * @param pageSize Page size.
+     * @return Next link for fragmented entries or {@code 0} if none.
+     * @throws IgniteCheckedException If failed.
+     */
+    public long removeRow(long pageAddr, int itemId, int pageSize) throws IgniteCheckedException {
+        assert checkIndex(itemId) : itemId;
+
+        final int dataOff = getDataOffset(pageAddr, itemId, pageSize);
+        final long nextLink = isFragmented(pageAddr, dataOff) ? getNextFragmentLink(pageAddr, dataOff) : 0;
+
+        // Record original counts to calculate delta in free space in the end of remove.
+        final int directCnt = getDirectCount(pageAddr);
+        final int indirectCnt = getIndirectCount(pageAddr);
+
+        int curIndirectCnt = indirectCnt;
+
+        assert directCnt > 0 : directCnt; // Direct count always represents overall number of live items.
+
+        // Remove the last item on the page.
+        if (directCnt == 1) {
+            assert (indirectCnt == 0 && itemId == 0) ||
+                (indirectCnt == 1 && itemId == itemId(getItem(pageAddr, 1))) : itemId;
+
+            setEmptyPage(pageAddr, pageSize);
+        }
+        else {
+            // Get the entry size before the actual remove.
+            int rmvEntrySize = getPageEntrySize(pageAddr, dataOff, SHOW_PAYLOAD_LEN | SHOW_LINK);
+
+            int indirectId = 0;
+
+            if (itemId >= directCnt) { // Need to remove indirect item.
+                assert indirectCnt > 0;
+
+                indirectId = findIndirectItemIndex(pageAddr, itemId, directCnt, indirectCnt);
+
+                assert indirectId >= directCnt;
+
+                itemId = directItemIndex(getItem(pageAddr, indirectId));
+
+                assert itemId < directCnt;
+            }
+
+            boolean dropLast = true;
+
+            if (itemId + 1 < directCnt) // It is not the last direct item.
+                dropLast = moveLastItem(pageAddr, itemId, directCnt, indirectCnt);
+
+            if (indirectId == 0) {// For the last direct item with no indirect item.
+                if (dropLast)
+                    moveItems(pageAddr, directCnt, indirectCnt, -1, pageSize);
+                else
+                    curIndirectCnt++;
+            }
+            else {
+                if (dropLast)
+                    moveItems(pageAddr, directCnt, indirectId - directCnt, -1, pageSize);
+
+                moveItems(pageAddr, indirectId + 1, directCnt + indirectCnt - indirectId - 1, dropLast ? -2 : -1, pageSize);
+
+                if (dropLast)
+                    curIndirectCnt--;
+            }
+
+            setIndirectCount(pageAddr, curIndirectCnt);
+            setDirectCount(pageAddr, directCnt - 1);
+
+            assert getIndirectCount(pageAddr) <= getDirectCount(pageAddr);
+
+            // Increase free space.
+            setRealFreeSpace(pageAddr,
+                getRealFreeSpace(pageAddr) + rmvEntrySize + ITEM_SIZE * (directCnt - getDirectCount(pageAddr) + indirectCnt - getIndirectCount(pageAddr)),
+                pageSize);
+        }
+
+        return nextLink;
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param idx Index.
+     * @param cnt Count.
+     * @param step Step.
+     * @param pageSize Page size.
+     */
+    private void moveItems(long pageAddr, int idx, int cnt, int step, int pageSize) {
+        assert cnt >= 0 : cnt;
+
+        if (cnt != 0)
+            moveBytes(pageAddr, itemOffset(idx), cnt * ITEM_SIZE, step * ITEM_SIZE, pageSize);
+    }
+
+    /**
+     * @param newEntryFullSize New entry full size (with item, length and link).
+     * @param firstEntryOff First entry data offset.
+     * @param directCnt Direct items count.
+     * @param indirectCnt Indirect items count.
+     * @return {@code true} If there is enough space for the entry.
+     */
+    private boolean isEnoughSpace(int newEntryFullSize, int firstEntryOff, int directCnt, int indirectCnt) {
+        return ITEMS_OFF + ITEM_SIZE * (directCnt + indirectCnt) <= firstEntryOff - newEntryFullSize;
+    }
+
+    /**
+     * Adds row to this data page and sets respective link to the given row object.
+     *
+     * @param pageAddr Page address.
+     * @param row Data row.
+     * @param rowSize Row size.
+     * @param pageSize Page size.
+     * @throws IgniteCheckedException If failed.
+     */
+    public void addRow(
+        final long pageId,
+        final long pageAddr,
+        T row,
+        final int rowSize,
+        final int pageSize
+    ) throws IgniteCheckedException {
+        assert rowSize <= getFreeSpace(pageAddr) : "can't call addRow if not enough space for the whole row";
+
+        int fullEntrySize = getPageEntrySize(rowSize, SHOW_PAYLOAD_LEN | SHOW_ITEM);
+
+        int directCnt = getDirectCount(pageAddr);
+        int indirectCnt = getIndirectCount(pageAddr);
+
+        int dataOff = getDataOffsetForWrite(pageAddr, fullEntrySize, directCnt, indirectCnt, pageSize);
+
+        writeRowData(pageAddr, dataOff, rowSize, row, true);
+
+        int itemId = addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
+
+        setLinkByPageId(row, pageId, itemId);
+    }
+
+    /**
+     * Adds row to this data page and sets respective link to the given row object.
+     *
+     * @param pageAddr Page address.
+     * @param payload Payload.
+     * @param pageSize Page size.
+     * @throws IgniteCheckedException If failed.
+     */
+    public void addRow(
+        long pageAddr,
+        byte[] payload,
+        int pageSize
+    ) throws IgniteCheckedException {
+        assert payload.length <= getFreeSpace(pageAddr) : "can't call addRow if not enough space for the whole row";
+
+        int fullEntrySize = getPageEntrySize(payload.length, SHOW_PAYLOAD_LEN | SHOW_ITEM);
+
+        int directCnt = getDirectCount(pageAddr);
+        int indirectCnt = getIndirectCount(pageAddr);
+
+        int dataOff = getDataOffsetForWrite(pageAddr, fullEntrySize, directCnt, indirectCnt, pageSize);
+
+        writeRowData(pageAddr, dataOff, payload);
+
+        addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param entryFullSize New entry full size (with item, length and link).
+     * @param directCnt Direct items count.
+     * @param indirectCnt Indirect items count.
+     * @param dataOff First entry offset.
+     * @param pageSize Page size.
+     * @return First entry offset after compaction.
+     */
+    private int compactIfNeed(
+        final long pageAddr,
+        final int entryFullSize,
+        final int directCnt,
+        final int indirectCnt,
+        int dataOff,
+        int pageSize
+    ) {
+        if (!isEnoughSpace(entryFullSize, dataOff, directCnt, indirectCnt)) {
+            dataOff = compactDataEntries(pageAddr, directCnt, pageSize);
+
+            assert isEnoughSpace(entryFullSize, dataOff, directCnt, indirectCnt);
+        }
+
+        return dataOff;
+    }
+
+    /**
+     * Put item reference on entry.
+     *
+     * @param pageAddr Page address.
+     * @param fullEntrySize Full entry size (with link, payload size and item).
+     * @param directCnt Direct items count.
+     * @param indirectCnt Indirect items count.
+     * @param dataOff Data offset.
+     * @param pageSize Page size.
+     * @return Item ID.
+     */
+    private int addItem(final long pageAddr,
+        final int fullEntrySize,
+        final int directCnt,
+        final int indirectCnt,
+        final int dataOff,
+        final int pageSize) {
+        setFirstEntryOffset(pageAddr, dataOff, pageSize);
+
+        int itemId = insertItem(pageAddr, dataOff, directCnt, indirectCnt, pageSize);
+
+        assert checkIndex(itemId) : itemId;
+        assert getIndirectCount(pageAddr) <= getDirectCount(pageAddr);
+
+        // Update free space. If number of indirect items changed, then we were able to reuse an item slot.
+        setRealFreeSpace(pageAddr,
+            getRealFreeSpace(pageAddr) - fullEntrySize + (getIndirectCount(pageAddr) != indirectCnt ? ITEM_SIZE : 0),
+            pageSize);
+
+        return itemId;
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param fullEntrySize Full entry size.
+     * @param directCnt Direct items count.
+     * @param indirectCnt Indirect items count.
+     * @param pageSize Page size.
+     * @return Offset in the buffer where the entry must be written.
+     */
+    private int getDataOffsetForWrite(long pageAddr, int fullEntrySize, int directCnt, int indirectCnt, int pageSize) {
+        int dataOff = getFirstEntryOffset(pageAddr);
+
+        // Compact if we do not have enough space for entry.
+        dataOff = compactIfNeed(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
+
+        // We will write data right before the first entry.
+        dataOff -= fullEntrySize - ITEM_SIZE;
+
+        return dataOff;
+    }
+
+    /**
+     * Adds maximum possible fragment of the given row to this data page and sets respective link to the row.
+     *
+     * @param pageMem Page memory.
+     * @param pageAddr Page address.
+     * @param row Data row.
+     * @param written Number of bytes of row size that was already written.
+     * @param rowSize Row size.
+     * @param pageSize Page size.
+     * @return Written payload size.
+     * @throws IgniteCheckedException If failed.
+     */
+    public int addRowFragment(
+        PageMemory pageMem,
+        long pageAddr,
+        T row,
+        int written,
+        int rowSize,
+        int pageSize
+    ) throws IgniteCheckedException {
+        return addRowFragment(pageMem, pageAddr, written, rowSize, row.link(), row, null, pageSize);
+    }
+
+    /**
+     * Adds this payload as a fragment to this data page.
+     *
+     * @param pageAddr Page address.
+     * @param payload Payload bytes.
+     * @param lastLink Link to the previous written fragment (link to the tail).
+     * @param pageSize Page size.
+     * @throws IgniteCheckedException If failed.
+     */
+    public void addRowFragment(
+        long pageAddr,
+        byte[] payload,
+        long lastLink,
+        int pageSize
+    ) throws IgniteCheckedException {
+        addRowFragment(null, pageAddr, 0, 0, lastLink, null, payload, pageSize);
+    }
+
+    /**
+     * Adds maximum possible fragment of the given row to this data page and sets respective link to the row.
+     *
+     * @param pageMem Page memory.
+     * @param pageAddr Page address.
+     * @param written Number of bytes of row size that was already written.
+     * @param rowSize Row size.
+     * @param lastLink Link to the previous written fragment (link to the tail).
+     * @param row Row.
+     * @param payload Payload bytes.
+     * @param pageSize Page size.
+     * @return Written payload size.
+     * @throws IgniteCheckedException If failed.
+     */
+    private int addRowFragment(
+        PageMemory pageMem,
+        long pageAddr,
+        int written,
+        int rowSize,
+        long lastLink,
+        T row,
+        byte[] payload,
+        int pageSize
+    ) throws IgniteCheckedException {
+        assert payload == null ^ row == null;
+
+        int directCnt = getDirectCount(pageAddr);
+        int indirectCnt = getIndirectCount(pageAddr);
+
+        int payloadSize = payload != null ? payload.length :
+            Math.min(rowSize - written, getFreeSpace(pageAddr));
+
+        int fullEntrySize = getPageEntrySize(payloadSize, SHOW_PAYLOAD_LEN | SHOW_LINK | SHOW_ITEM);
+        int dataOff = getDataOffsetForWrite(pageAddr, fullEntrySize, directCnt, indirectCnt, pageSize);
+
+        if (payload == null) {
+            ByteBuffer buf = pageMem.pageBuffer(pageAddr);
+
+            buf.position(dataOff);
+
+            short p = (short)(payloadSize | FRAGMENTED_FLAG);
+
+            buf.putShort(p);
+            buf.putLong(lastLink);
+
+            int rowOff = rowSize - written - payloadSize;
+
+            writeFragmentData(row, buf, rowOff, payloadSize);
+        }
+        else {
+            PageUtils.putShort(pageAddr, dataOff, (short)(payloadSize | FRAGMENTED_FLAG));
+
+            PageUtils.putLong(pageAddr, dataOff + 2, lastLink);
+
+            PageUtils.putBytes(pageAddr, dataOff + 10, payload);
+        }
+
+        int itemId = addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
+
+        if (row != null)
+            setLink(row, pageAddr, itemId);
+
+        return payloadSize;
+    }
+
+    /**
+     * @param row Row to set link to.
+     * @param pageAddr Page address.
+     * @param itemId Item ID.
+     */
+    private void setLink(T row, long pageAddr, int itemId) {
+        long pageId = getPageId(pageAddr);
+
+        setLinkByPageId(row, pageId, itemId);
+    }
+
+    /**
+     * @param row Row to set link to.
+     * @param pageId Page ID.
+     * @param itemId Item ID.
+     */
+    private void setLinkByPageId(T row, long pageId, int itemId) {
+        row.link(PageIdUtils.link(pageId, itemId));
+    }
+
+    /**
+     * Write row data fragment.
+     *
+     * @param row Row.
+     * @param buf Byte buffer.
+     * @param rowOff Offset in row data bytes.
+     * @param payloadSize Data length that should be written in a fragment.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected abstract void writeFragmentData(
+        final T row,
+        final ByteBuffer buf,
+        final int rowOff,
+        final int payloadSize
+    ) throws IgniteCheckedException;
+
+    /**
+     * @param pageAddr Page address.
+     * @param dataOff Data offset.
+     * @param directCnt Direct items count.
+     * @param indirectCnt Indirect items count.
+     * @param pageSize Page size.
+     * @return Item ID (insertion index).
+     */
+    private int insertItem(long pageAddr, int dataOff, int directCnt, int indirectCnt, int pageSize) {
+        if (indirectCnt > 0) {
+            // If the first indirect item is on correct place to become the last direct item, do the transition
+            // and insert the new item into the free slot which was referenced by this first indirect item.
+            short item = getItem(pageAddr, directCnt);
+
+            if (itemId(item) == directCnt) {
+                int directItemIdx = directItemIndex(item);
+
+                setItem(pageAddr, directCnt, getItem(pageAddr, directItemIdx));
+                setItem(pageAddr, directItemIdx, directItemFromOffset(dataOff));
+
+                setDirectCount(pageAddr, directCnt + 1);
+                setIndirectCount(pageAddr, indirectCnt - 1);
+
+                return directItemIdx;
+            }
+        }
+
+        // Move all the indirect items forward to make a free slot and insert new item at the end of direct items.
+        moveItems(pageAddr, directCnt, indirectCnt, +1, pageSize);
+
+        setItem(pageAddr, directCnt, directItemFromOffset(dataOff));
+
+        setDirectCount(pageAddr, directCnt + 1);
+        assert getDirectCount(pageAddr) == directCnt + 1;
+
+        return directCnt; // Previous directCnt will be our itemId.
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param directCnt Direct items count.
+     * @param pageSize Page size.
+     * @return New first entry offset.
+     */
+    private int compactDataEntries(long pageAddr, int directCnt, int pageSize) {
+        assert checkCount(directCnt) : directCnt;
+
+        int[] offs = new int[directCnt];
+
+        for (int i = 0; i < directCnt; i++) {
+            int off = directItemToOffset(getItem(pageAddr, i));
+
+            offs[i] = (off << 8) | i; // This way we'll be able to sort by offset using Arrays.sort(...).
+        }
+
+        Arrays.sort(offs);
+
+        // Move right all of the entries if possible to make the page as compact as possible to its tail.
+        int prevOff = pageSize;
+
+        final int start = directCnt - 1;
+        int curOff = offs[start] >>> 8;
+        int curEntrySize = getPageEntrySize(pageAddr, curOff, SHOW_PAYLOAD_LEN | SHOW_LINK);
+
+        for (int i = start; i >= 0; i--) {
+            assert curOff < prevOff : curOff;
+
+            int delta = prevOff - (curOff + curEntrySize);
+
+            int off = curOff;
+            int entrySize = curEntrySize;
+
+            if (delta != 0) { // Move right.
+                assert delta > 0 : delta;
+
+                int itemId = offs[i] & 0xFF;
+
+                setItem(pageAddr, itemId, directItemFromOffset(curOff + delta));
+
+                for (int j = i - 1; j >= 0; j--) {
+                    int offNext = offs[j] >>> 8;
+                    int nextSize = getPageEntrySize(pageAddr, offNext, SHOW_PAYLOAD_LEN | SHOW_LINK);
+
+                    if (offNext + nextSize == off) {
+                        i--;
+
+                        off = offNext;
+                        entrySize += nextSize;
+
+                        itemId = offs[j] & 0xFF;
+                        setItem(pageAddr, itemId, directItemFromOffset(offNext + delta));
+                    }
+                    else {
+                        curOff = offNext;
+                        curEntrySize = nextSize;
+
+                        break;
+                    }
+                }
+
+                moveBytes(pageAddr, off, entrySize, delta, pageSize);
+
+                off += delta;
+            }
+            else if (i > 0) {
+                curOff = offs[i - 1] >>> 8;
+                curEntrySize = getPageEntrySize(pageAddr, curOff, SHOW_PAYLOAD_LEN | SHOW_LINK);
+            }
+
+            prevOff = off;
+        }
+
+        return prevOff;
+    }
+
+    /**
+     * Full-scan free space calculation procedure.
+     *
+     * @param pageAddr Page to scan.
+     * @param pageSize Page size.
+     * @return Actual free space in the buffer.
+     */
+    private int actualFreeSpace(long pageAddr, int pageSize) {
+        int directCnt = getDirectCount(pageAddr);
+
+        int entriesSize = 0;
+
+        for (int i = 0; i < directCnt; i++) {
+            int off = directItemToOffset(getItem(pageAddr, i));
+
+            int entrySize = getPageEntrySize(pageAddr, off, SHOW_PAYLOAD_LEN | SHOW_LINK);
+
+            entriesSize += entrySize;
+        }
+
+        return pageSize - ITEMS_OFF - entriesSize - (directCnt + getIndirectCount(pageAddr)) * ITEM_SIZE;
+    }
+
+    /**
+     * @param addr Address.
+     * @param off Offset.
+     * @param cnt Count.
+     * @param step Step.
+     * @param pageSize Page size.
+     */
+    private void moveBytes(long addr, int off, int cnt, int step, int pageSize) {
+        assert step != 0 : step;
+        assert off + step >= 0;
+        assert off + step + cnt <= pageSize : "[off=" + off + ", step=" + step + ", cnt=" + cnt +
+            ", cap=" + pageSize + ']';
+
+        PageHandler.copyMemory(addr, off, addr, off + step, cnt);
+    }
+
+    /**
+     * @param pageAddr Page address.
+     * @param dataOff Data offset.
+     * @param payloadSize Payload size.
+     * @param row Data row.
+     * @param newRow {@code False} if existing cache entry is updated, in this case skip key data write.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected abstract void writeRowData(
+        long pageAddr,
+        int dataOff,
+        int payloadSize,
+        T row,
+        boolean newRow
+    ) throws IgniteCheckedException;
+
+    /**
+     * @param pageAddr Page address.
+     * @param dataOff Data offset.
+     * @param payload Payload
+     */
+    protected void writeRowData(
+        long pageAddr,
+        int dataOff,
+        byte[] payload
+    ) {
+        PageUtils.putShort(pageAddr, dataOff, (short)payload.length);
+        dataOff += 2;
+
+        PageUtils.putBytes(pageAddr, dataOff, payload);
+    }
+
+    /**
+     * @param row Row.
+     * @return Row size in page.
+     * @throws IgniteCheckedException if failed.
+     */
+    public abstract int getRowSize(T row) throws IgniteCheckedException;
+
+    /**
+     * Defines closure interface for applying computations to data page items.
+     *
+     * @param <T> Closure return type.
+     */
+    public interface CC<T> {
+        /**
+         * Closure body.
+         *
+         * @param link Link to item.
+         * @return Closure return value.
+         * @throws IgniteCheckedException In case of error in closure body.
+         */
+        public T apply(long link) throws IgniteCheckedException;
+    }
+}


[04/17] ignite git commit: IEP-4 Baseline topology for persistent caches (Phase 1) Contributed by: Dmitriy Govorukhin Dmitry Pavlov Eduard Shangareev Ily

Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteBaselineAffinityTopologyActivationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteBaselineAffinityTopologyActivationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteBaselineAffinityTopologyActivationTest.java
new file mode 100644
index 0000000..1a9393a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteBaselineAffinityTopologyActivationTest.java
@@ -0,0 +1,993 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.cluster.DetachedClusterNode;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
+import org.apache.ignite.internal.processors.cluster.BaselineTopologyHistory;
+import org.apache.ignite.internal.processors.cluster.BaselineTopologyHistoryItem;
+import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+/**
+ *
+ */
+public class IgniteBaselineAffinityTopologyActivationTest extends GridCommonAbstractTest {
+    /** */
+    private String consId;
+
+    /** Entries count to add to cache. */
+    private static final int ENTRIES_COUNT = 100;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        if (consId != null)
+            cfg.setConsistentId(consId);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration().setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setPersistenceEnabled(true).setMaxSize(10 * 1024 * 1024)
+
+            ).setWalMode(WALMode.LOG_ONLY)
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids(false);
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /**
+     * Verifies that when old but compatible node
+     * (it is the node that once wasn't presented in branchingHistory but hasn't participated in any branching point)
+     * joins the cluster after restart, cluster gets activated.
+     */
+    public void testAutoActivationWithCompatibleOldNode() throws Exception {
+        startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        startGridWithConsistentId("C").active(true);
+
+        stopAllGrids(false);
+
+        startGridWithConsistentId("A");
+        startGridWithConsistentId("B").active(true);
+
+        {
+            IgniteEx nodeA = grid("A");
+
+            assertNotNull(nodeA.cluster().currentBaselineTopology());
+            assertEquals(3, nodeA.cluster().currentBaselineTopology().size());
+
+            assertTrue(nodeA.cluster().active());
+        }
+
+        stopAllGrids(false);
+
+        startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+
+        {
+            IgniteEx nodeA = grid("A");
+
+            assertNotNull(nodeA.cluster().currentBaselineTopology());
+            assertEquals(3, nodeA.cluster().currentBaselineTopology().size());
+
+            assertFalse(nodeA.cluster().active());
+        }
+
+        final Ignite nodeC = startGridWithConsistentId("C");
+
+        boolean active = GridTestUtils.waitForCondition(
+            new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    return nodeC.active();
+                }
+            },
+            10_000
+        );
+
+        assertTrue(active);
+    }
+
+    /**
+     * IgniteCluster::setBaselineTopology(long topVer) should throw an exception
+     * when online node from current BaselineTopology is not presented in topology version.
+     */
+    public void testBltChangeTopVerRemoveOnlineNodeFails() throws Exception {
+        Ignite ignite = startGridWithConsistentId("A");
+
+        ignite.active(true);
+
+        long singleNodeTopVer = ignite.cluster().topologyVersion();
+
+        startGridWithConsistentId("OnlineConsID");
+
+        ignite.cluster().setBaselineTopology(baselineNodes(ignite.cluster().forServers().nodes()));
+
+        boolean expectedExceptionThrown = false;
+
+        try {
+            ignite.cluster().setBaselineTopology(singleNodeTopVer);
+        }
+        catch (IgniteException e) {
+            String errMsg = e.getMessage();
+            assertTrue(errMsg.startsWith("Removing online nodes"));
+            assertTrue(errMsg.contains("[OnlineConsID]"));
+
+            expectedExceptionThrown = true;
+        }
+
+        assertTrue(expectedExceptionThrown);
+    }
+
+    /**
+     * Verifies that online nodes cannot be removed from BaselineTopology (this may change in future).
+     */
+    public void testOnlineNodesCannotBeRemovedFromBaselineTopology() throws Exception {
+        Ignite nodeA = startGridWithConsistentId("A");
+        Ignite nodeB = startGridWithConsistentId("B");
+        Ignite nodeC = startGridWithConsistentId("OnlineConsID");
+
+        nodeC.active(true);
+
+        boolean expectedExceptionIsThrown = false;
+
+        try {
+            nodeC.cluster().setBaselineTopology(Arrays.asList((BaselineNode) nodeA.cluster().localNode(),
+                nodeB.cluster().localNode()));
+        } catch (IgniteException e) {
+            String errMsg = e.getMessage();
+            assertTrue(errMsg.startsWith("Removing online nodes"));
+            assertTrue(errMsg.contains("[OnlineConsID]"));
+
+            expectedExceptionIsThrown = true;
+        }
+
+        assertTrue(expectedExceptionIsThrown);
+    }
+
+    /**
+     *
+     */
+    public void testNodeFailsToJoinWithIncompatiblePreviousBaselineTopology() throws Exception {
+        startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        Ignite nodeC = startGridWithConsistentId("C");
+
+        nodeC.active(true);
+
+        stopAllGrids(false);
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        startGridWithConsistentId("B").active(true);
+
+        nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+        stopAllGrids(false);
+
+        startGridWithConsistentId("C").active(true);
+
+        stopGrid("C", false);
+
+        startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+
+        boolean expectedExceptionThrown = false;
+
+        try {
+            startGridWithConsistentId("C");
+        }
+        catch (IgniteCheckedException e) {
+            expectedExceptionThrown = true;
+
+            if (e.getCause() != null && e.getCause().getCause() != null) {
+                Throwable rootCause = e.getCause().getCause();
+
+                if (!(rootCause instanceof IgniteSpiException) || !rootCause.getMessage().contains("not compatible"))
+                    Assert.fail("Unexpected ignite exception was thrown: " + e);
+            }
+            else
+                throw e;
+        }
+
+        assertTrue("Expected exception wasn't thrown.", expectedExceptionThrown);
+    }
+
+    /**
+     * Verifies scenario when parts of grid were activated independently they are not allowed to join
+     * into the same grid again (due to risks of incompatible data modifications).
+     */
+    public void testIncompatibleBltNodeIsProhibitedToJoinCluster() throws Exception {
+        startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        startGridWithConsistentId("C").active(true);
+
+        stopAllGrids(false);
+
+        startGridWithConsistentId("A");
+        startGridWithConsistentId("B").active(true);
+
+        stopAllGrids(false);
+
+        startGridWithConsistentId("C").active(true);
+
+        stopAllGrids(false);
+
+        startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+
+        boolean expectedExceptionThrown = false;
+
+        try {
+            startGridWithConsistentId("C");
+        }
+        catch (IgniteCheckedException e) {
+            expectedExceptionThrown = true;
+
+            if (e.getCause() != null && e.getCause().getCause() != null) {
+                Throwable rootCause = e.getCause().getCause();
+
+                if (!(rootCause instanceof IgniteSpiException) || !rootCause.getMessage().contains("not compatible"))
+                    Assert.fail("Unexpected ignite exception was thrown: " + e);
+            }
+            else
+                throw e;
+        }
+
+        assertTrue("Expected exception wasn't thrown.", expectedExceptionThrown);
+    }
+
+    /**
+     * Test verifies that node with out-of-data but still compatible Baseline Topology is allowed to join the cluster.
+     */
+    public void testNodeWithOldBltIsAllowedToJoinCluster() throws Exception {
+        final long expectedHash1 = (long)"A".hashCode() + "B".hashCode() + "C".hashCode();
+
+        BaselineTopologyVerifier verifier1 = new BaselineTopologyVerifier() {
+            @Override public void verify(BaselineTopology blt) {
+                assertNotNull(blt);
+
+                assertEquals(3, blt.consistentIds().size());
+
+                long activationHash = U.field(blt, "branchingPntHash");
+
+                assertEquals(expectedHash1, activationHash);
+            }
+        };
+
+        final long expectedHash2 = (long)"A".hashCode() + "B".hashCode();
+
+        BaselineTopologyVerifier verifier2 = new BaselineTopologyVerifier() {
+            @Override public void verify(BaselineTopology blt) {
+                assertNotNull(blt);
+
+                assertEquals(3, blt.consistentIds().size());
+
+                long activationHash = U.field(blt, "branchingPntHash");
+
+                assertEquals(expectedHash2, activationHash);
+            }
+        };
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        Ignite nodeB = startGridWithConsistentId("B");
+        Ignite nodeC = startGridWithConsistentId("C");
+
+        nodeC.active(true);
+        verifyBaselineTopologyOnNodes(verifier1, new Ignite[] {nodeA, nodeB, nodeC});
+
+        stopAllGrids(false);
+
+        nodeA = startGridWithConsistentId("A");
+        nodeB = startGridWithConsistentId("B");
+
+        nodeB.active(true);
+
+        verifyBaselineTopologyOnNodes(verifier2, new Ignite[] {nodeA, nodeB});
+
+        stopAllGrids(false);
+
+        nodeA = startGridWithConsistentId("A");
+        nodeB = startGridWithConsistentId("B");
+        nodeC = startGridWithConsistentId("C");
+
+        verifyBaselineTopologyOnNodes(verifier2, new Ignite[] {nodeA, nodeB, nodeC});
+    }
+
+    /**
+     * Verifies that when new node outside of baseline topology joins active cluster with BLT already set
+     * it receives BLT from the cluster and stores it locally.
+     */
+    public void testNewNodeJoinsToActiveCluster() throws Exception {
+        Ignite nodeA = startGridWithConsistentId("A");
+        Ignite nodeB = startGridWithConsistentId("B");
+        Ignite nodeC = startGridWithConsistentId("C");
+
+        nodeC.active(true);
+
+        BaselineTopologyVerifier verifier1 = new BaselineTopologyVerifier() {
+            @Override public void verify(BaselineTopology blt) {
+                assertNotNull(blt);
+
+                assertEquals(3, blt.consistentIds().size());
+            }
+        };
+
+        verifyBaselineTopologyOnNodes(verifier1, new Ignite[] {nodeA, nodeB, nodeC});
+
+        Ignite nodeD = startGridWithConsistentId("D");
+
+        verifyBaselineTopologyOnNodes(verifier1, new Ignite[] {nodeD});
+
+        stopAllGrids(false);
+
+        nodeD = startGridWithConsistentId("D");
+
+        assertFalse(nodeD.active());
+
+        verifyBaselineTopologyOnNodes(verifier1, new Ignite[] {nodeD});
+    }
+
+    /**
+     *
+     */
+    public void testRemoveNodeFromBaselineTopology() throws Exception {
+        final long expectedActivationHash = (long)"A".hashCode() + "C".hashCode();
+
+        BaselineTopologyVerifier verifier = new BaselineTopologyVerifier() {
+            @Override public void verify(BaselineTopology blt) {
+                assertNotNull(blt);
+
+                assertEquals(2, blt.consistentIds().size());
+
+                long activationHash = U.field(blt, "branchingPntHash");
+
+                assertEquals(expectedActivationHash, activationHash);
+            }
+        };
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        Ignite nodeC = startGridWithConsistentId("C");
+
+        nodeC.active(true);
+
+        stopGrid("B", false);
+
+        nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+        boolean activated = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return grid("A").active();
+            }
+        }, 10_000);
+
+        assertEquals(true, activated);
+
+        verifyBaselineTopologyOnNodes(verifier, new Ignite[] {nodeA, nodeC});
+
+        stopAllGrids(false);
+
+        nodeA = startGridWithConsistentId("A");
+        nodeC = startGridWithConsistentId("C");
+
+        activated = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return grid("A").active();
+            }
+        }, 10_000);
+
+        assertTrue(activated);
+
+        verifyBaselineTopologyOnNodes(verifier, new Ignite[] {nodeA, nodeC});
+    }
+
+    /**
+     *
+     */
+    public void testAddNodeToBaselineTopology() throws Exception {
+        final long expectedActivationHash = (long)"A".hashCode() + "B".hashCode() + "C".hashCode() + "D".hashCode();
+
+        BaselineTopologyVerifier verifier = new BaselineTopologyVerifier() {
+            @Override public void verify(BaselineTopology blt) {
+                assertNotNull(blt);
+
+                assertEquals(4, blt.consistentIds().size());
+
+                long activationHash = U.field(blt, "branchingPntHash");
+
+                assertEquals(expectedActivationHash, activationHash);
+            }
+        };
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        Ignite nodeB = startGridWithConsistentId("B");
+        Ignite nodeC = startGridWithConsistentId("C");
+
+        nodeC.active(true);
+
+        IgniteEx nodeD = (IgniteEx) startGridWithConsistentId("D");
+
+        nodeD.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+        verifyBaselineTopologyOnNodes(verifier, new Ignite[]{nodeA, nodeB, nodeC, nodeD});
+    }
+
+    /**
+     * Verifies that baseline topology is removed successfully through baseline changing API.
+     */
+    public void testRemoveBaselineTopology() throws Exception {
+        BaselineTopologyVerifier verifier = new BaselineTopologyVerifier() {
+            @Override public void verify(BaselineTopology blt) {
+                assertNull(blt);
+            }
+        };
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        Ignite nodeB = startGridWithConsistentId("B");
+        Ignite nodeC = startGridWithConsistentId("C");
+
+        nodeA.active(true);
+
+        nodeA.cluster().setBaselineTopology(null);
+
+        verifyBaselineTopologyOnNodes(verifier, new Ignite[] {nodeA, nodeB, nodeC});
+    }
+
+    /** */
+    private interface BaselineTopologyVerifier {
+        /** */
+        public void verify(BaselineTopology blt);
+    }
+
+    /** */
+    private interface BaselineTopologyHistoryVerifier {
+        /** */
+        public void verify(BaselineTopologyHistory bltHist);
+    }
+
+    /** */
+    private void verifyBaselineTopologyOnNodes(BaselineTopologyVerifier bltVerifier, Ignite[] igs) {
+        for (Ignite ig : igs) {
+            BaselineTopology blt = getBaselineTopology(ig);
+
+            bltVerifier.verify(blt);
+        }
+    }
+
+    /** */
+    private void verifyBaselineTopologyHistoryOnNodes(BaselineTopologyHistoryVerifier bltHistVerifier, Ignite[] igs) {
+        for (Ignite ig : igs) {
+            BaselineTopologyHistory blt = getBaselineTopologyHistory(ig);
+
+            bltHistVerifier.verify(blt);
+        }
+    }
+
+    /** */
+    private Ignite startGridWithConsistentId(String consId) throws Exception {
+        this.consId = consId;
+
+        return startGrid(consId);
+    }
+
+    /**
+     * Verifies that when new node joins already active cluster and new activation request is issued,
+     * no changes to BaselineTopology branching history happen.
+     */
+    public void testActivationHashIsNotUpdatedOnMultipleActivationRequests() throws Exception {
+        final long expectedActivationHash = (long)"A".hashCode();
+
+        BaselineTopologyVerifier verifier = new BaselineTopologyVerifier() {
+            @Override public void verify(BaselineTopology blt) {
+                long activationHash = U.field(blt, "branchingPntHash");
+
+                assertEquals(expectedActivationHash, activationHash);
+            }
+        };
+
+        Ignite nodeA = startGridWithConsistentId("A");
+
+        nodeA.active(true);
+
+        Ignite nodeB = startGridWithConsistentId("B");
+
+        nodeA.active(true);
+
+        verifyBaselineTopologyOnNodes(verifier, new Ignite[] {nodeA, nodeB});
+    }
+
+    /**
+     * Verifies that grid is autoactivated when full BaselineTopology is preset even on one node
+     * and then all other nodes from BaselineTopology are started.
+     */
+    public void testAutoActivationWithBaselineTopologyPreset() throws Exception {
+        Ignite ig = startGridWithConsistentId("A");
+
+        ig.active(true);
+
+        ig.cluster().setBaselineTopology(Arrays.asList(new BaselineNode[] {
+            createBaselineNodeWithConsId("A"), createBaselineNodeWithConsId("B"), createBaselineNodeWithConsId("C")}));
+
+        stopAllGrids();
+
+        final Ignite ig1 = startGridWithConsistentId("A");
+
+        startGridWithConsistentId("B");
+
+        startGridWithConsistentId("C");
+
+        boolean activated = GridTestUtils.waitForCondition(
+            new GridAbsPredicate() {
+               @Override public boolean apply() {
+                   return ig1.active();
+               }
+            },
+            10_000
+        );
+
+        assertTrue(activated);
+    }
+
+    /**
+     * Creates BaselineNode with specific attribute indicating that this node is not client.
+     */
+    private BaselineNode createBaselineNodeWithConsId(String consId) {
+        Map<String, Object> attrs = new HashMap<>();
+
+        attrs.put("org.apache.ignite.cache.client", false);
+
+        return new DetachedClusterNode(consId, attrs);
+    }
+
+    /** */
+    public void testAutoActivationSimple() throws Exception {
+        startGrids(3);
+
+        IgniteEx srv = grid(0);
+
+        srv.active(true);
+
+        createAndFillCache(srv);
+
+        // TODO: final implementation should work with cancel == true.
+        stopAllGrids();
+
+        //note: no call for activation after grid restart
+        startGrids(3);
+
+        final Ignite ig = grid(0);
+
+        boolean clusterActive = GridTestUtils.waitForCondition(
+            new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    return ig.active();
+                }
+            },
+            10_000);
+
+        assertTrue(clusterActive);
+
+        checkDataInCache((IgniteEx) ig);
+    }
+
+    /**
+     *
+     */
+    public void testNoAutoActivationOnJoinNewNodeToInactiveCluster() throws Exception {
+        startGrids(2);
+
+        IgniteEx srv = grid(0);
+
+        srv.active(true);
+
+        awaitPartitionMapExchange();
+
+        assertTrue(srv.active());
+
+        srv.active(false);
+
+        assertFalse(srv.active());
+
+        startGrid(2);
+
+        Thread.sleep(3_000);
+
+        assertFalse(srv.active());
+    }
+
+    /**
+     * Verifies that neither BaselineTopology nor BaselineTopologyHistory are changed when cluster is deactivated.
+     */
+    public void testBaselineTopologyRemainsTheSameOnClusterDeactivation() throws Exception {
+        startGrids(2);
+
+        IgniteEx srv = grid(0);
+
+        srv.active(true);
+
+        awaitPartitionMapExchange();
+
+        assertTrue(srv.active());
+
+        srv.active(false);
+
+        BaselineTopology blt = getBaselineTopology(srv);
+
+        BaselineTopologyHistory bltHist = getBaselineTopologyHistory(srv);
+
+        assertEquals(0, blt.id());
+
+        assertEquals(2, blt.consistentIds().size());
+
+        assertEquals(1, blt.branchingHistory().size());
+
+        assertEquals(0, bltHist.history().size());
+    }
+
+    /**
+     *
+     */
+    public void testBaselineHistorySyncWithNewNode() throws Exception {
+        final long expectedBranchingHash = "A".hashCode() + "B".hashCode() + "C".hashCode();
+
+        BaselineTopologyHistoryVerifier verifier = new BaselineTopologyHistoryVerifier() {
+            @Override public void verify(BaselineTopologyHistory bltHist) {
+                assertNotNull(bltHist);
+
+                assertEquals(1, bltHist.history().size());
+
+                BaselineTopologyHistoryItem histItem = bltHist.history().get(0);
+
+                assertEquals(1, histItem.branchingHistory().size());
+
+                long actualBranchingHash = histItem.branchingHistory().get(0);
+
+                assertEquals(expectedBranchingHash, actualBranchingHash);
+            }
+        };
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        startGridWithConsistentId("C");
+
+        nodeA.active(true);
+
+        stopGrid("C", false);
+
+        nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+        startGridWithConsistentId("D");
+
+        stopAllGrids(false);
+
+        Ignite nodeD = startGridWithConsistentId("D");
+
+        verifyBaselineTopologyHistoryOnNodes(verifier, new Ignite[] {nodeD});
+    }
+
+    /**
+     *
+     */
+    public void testBaselineHistorySyncWithOldNodeWithCompatibleHistory() throws Exception {
+        final long expectedBranchingHash0 = "A".hashCode() + "B".hashCode() + "C".hashCode();
+
+        final long expectedBranchingHash1 = "A".hashCode() + "B".hashCode();
+
+        BaselineTopologyHistoryVerifier verifier = new BaselineTopologyHistoryVerifier() {
+            @Override public void verify(BaselineTopologyHistory bltHist) {
+                assertNotNull(bltHist);
+
+                assertEquals(2, bltHist.history().size());
+
+                BaselineTopologyHistoryItem histItem = bltHist.history().get(0);
+
+                assertEquals(1, histItem.branchingHistory().size());
+
+                long actualBranchingHash0 = histItem.branchingHistory().get(0);
+
+                assertEquals(expectedBranchingHash0, actualBranchingHash0);
+
+                histItem = bltHist.history().get(1);
+
+                assertEquals(1, histItem.branchingHistory().size());
+
+                long actualBranchingHash1 = histItem.branchingHistory().get(0);
+
+                assertEquals(expectedBranchingHash1, actualBranchingHash1);
+            }
+        };
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        startGridWithConsistentId("C");
+
+        nodeA.active(true);
+
+        stopGrid("C", false);
+
+        nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+        stopGrid("B", false);
+
+        nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+        startGridWithConsistentId("B");
+
+        stopAllGrids(false);
+
+        startGridWithConsistentId("A");
+        Ignite nodeB = startGridWithConsistentId("B");
+
+        verifyBaselineTopologyHistoryOnNodes(verifier, new Ignite[] {nodeB});
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testBaselineNotDeletedOnDeactivation() throws Exception {
+        Ignite nodeA = startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        startGridWithConsistentId("C");
+
+        nodeA.active(true);
+
+        assertNotNull(nodeA.cluster().currentBaselineTopology());
+
+        nodeA.active(false);
+
+        stopAllGrids();
+
+        nodeA = startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        startGridWithConsistentId("C");
+
+        final Ignite ig = nodeA;
+
+        boolean clusterActive = GridTestUtils.waitForCondition(
+            new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    return ig.active();
+                }
+            },
+            10_000);
+
+        assertNotNull(nodeA.cluster().currentBaselineTopology());
+
+        assertTrue(clusterActive);
+    }
+
+    /**
+     *
+     */
+    public void testNodeWithBltIsProhibitedToJoinNewCluster() throws Exception {
+        BaselineTopologyVerifier nullVerifier = new BaselineTopologyVerifier() {
+            @Override public void verify(BaselineTopology blt) {
+                assertNull(blt);
+            }
+        };
+
+        Ignite nodeC = startGridWithConsistentId("C");
+
+        nodeC.active(true);
+
+        stopGrid("C", false);
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        Ignite nodeB = startGridWithConsistentId("B");
+
+        verifyBaselineTopologyOnNodes(nullVerifier, new Ignite[] {nodeA, nodeB});
+
+        boolean expectedExceptionThrown = false;
+
+        try {
+            startGridWithConsistentId("C");
+        }
+        catch (IgniteCheckedException e) {
+            expectedExceptionThrown = true;
+
+            if (e.getCause() != null && e.getCause().getCause() != null) {
+                Throwable rootCause = e.getCause().getCause();
+
+                if (!(rootCause instanceof IgniteSpiException) || !rootCause.getMessage().contains("Node with set up BaselineTopology"))
+                    Assert.fail("Unexpected ignite exception was thrown: " + e);
+            }
+            else
+                throw e;
+        }
+
+        assertTrue("Expected exception wasn't thrown.", expectedExceptionThrown);
+    }
+
+    /**
+     * Restore this test when requirements for BaselineTopology deletion are clarified and this feature
+     * is covered with more tests.
+     */
+    public void _testBaselineTopologyHistoryIsDeletedOnBaselineDelete() throws Exception {
+        BaselineTopologyHistoryVerifier verifier = new BaselineTopologyHistoryVerifier() {
+            @Override public void verify(BaselineTopologyHistory bltHist) {
+                assertNotNull(bltHist);
+
+                assertEquals(0, bltHist.history().size());
+            }
+        };
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        startGridWithConsistentId("C");
+
+        nodeA.active(true);
+
+        stopAllGrids(false);
+
+        nodeA = startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+
+        nodeA.active(true);
+
+        nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+        stopAllGrids(false);
+
+        nodeA = startGridWithConsistentId("A");
+
+        nodeA.active(true);
+
+        nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+        stopAllGrids(false);
+
+        final Ignite node = startGridWithConsistentId("A");
+
+        boolean activated = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return node.active();
+            }
+        }, 10_000);
+
+        assertTrue(activated);
+
+        node.cluster().setBaselineTopology(null);
+
+        verifyBaselineTopologyHistoryOnNodes(verifier, new Ignite[] {node});
+
+        stopAllGrids(false);
+
+        nodeA = startGridWithConsistentId("A");
+
+        verifyBaselineTopologyHistoryOnNodes(verifier, new Ignite[] {nodeA});
+    }
+
+    /**
+     * Retrieves baseline topology from ignite node instance.
+     *
+     * @param ig Ig.
+     */
+    private BaselineTopology getBaselineTopology(Ignite ig) {
+        return ((DiscoveryDataClusterState) U.field(
+            (Object) U.field(
+                (Object) U.field(ig, "ctx"),
+                "stateProc"),
+            "globalState"))
+            .baselineTopology();
+    }
+
+    /** */
+    private BaselineTopologyHistory getBaselineTopologyHistory(Ignite ig) {
+        return U.field(
+            (Object) U.field(
+                (Object) U.field(ig, "ctx"),
+                "stateProc"),
+            "bltHist");
+    }
+
+    /** */
+    private void checkDataInCache(IgniteEx srv) {
+        IgniteCache<Object, Object> cache = srv.cache(DEFAULT_CACHE_NAME);
+
+        for (int i = 0; i < ENTRIES_COUNT; i++) {
+            TestValue testVal = (TestValue) cache.get(i);
+
+            assertNotNull(testVal);
+
+            assertEquals(i, testVal.id);
+        }
+    }
+
+    /** */
+    private void createAndFillCache(Ignite srv) {
+        IgniteCache cache = srv.getOrCreateCache(cacheConfiguration());
+
+        for (int i = 0; i < ENTRIES_COUNT; i++)
+            cache.put(i, new TestValue(i, "str" + i));
+    }
+
+    /** */
+    private CacheConfiguration cacheConfiguration() {
+        return new CacheConfiguration()
+            .setName(DEFAULT_CACHE_NAME)
+            .setCacheMode(CacheMode.PARTITIONED)
+            .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+            .setBackups(2)
+            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+    }
+
+    /** */
+    private static final class TestValue {
+        /** */
+        private final int id;
+
+        /** */
+        private final String strId;
+
+        /** */
+        private TestValue(int id, String strId) {
+            this.id = id;
+            this.strId = strId;
+        }
+    }
+
+    /** */
+    private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+        Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+        for (ClusterNode clN : clNodes)
+            res.add(clN);
+
+        return res;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java
index 50d7e7e..73c269a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java
@@ -17,6 +17,11 @@
 package org.apache.ignite.internal.processors.cache.persistence;
 
 import java.io.File;
+import java.nio.file.CopyOption;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.util.Arrays;
 import java.util.Collection;
 import org.apache.ignite.Ignite;
@@ -35,6 +40,7 @@ import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -56,12 +62,21 @@ public class IgnitePdsBinaryMetadataOnClusterRestartTest extends GridCommonAbstr
     private static final String DYNAMIC_STR_FIELD_NAME = "strField";
 
     /** */
+    private static final String CUSTOM_WORK_DIR_NAME_PATTERN = "node%s_workDir";
+
+    /** */
     private boolean clientMode;
 
+    /** */
+    private String customWorkSubDir;
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
+        if (customWorkSubDir != null)
+            cfg.setWorkDirectory(Paths.get(U.defaultWorkDirectory(), customWorkSubDir).toString());
+
         cfg.setClientMode(clientMode);
 
         cfg.setDataStorageConfiguration(
@@ -99,6 +114,257 @@ public class IgnitePdsBinaryMetadataOnClusterRestartTest extends GridCommonAbstr
     }
 
     /**
+     * @see <a href="https://issues.apache.org/jira/browse/IGNITE-7258">IGNITE-7258</a> refer to the following JIRA for more context about the problem verified by the test.
+     */
+    public void testUpdatedBinaryMetadataIsPreservedOnJoinToOldCoordinator() throws Exception {
+        Ignite ignite0 = startGridInASeparateWorkDir("A");
+        Ignite ignite1 = startGridInASeparateWorkDir("B");
+
+        ignite0.active(true);
+
+        IgniteCache<Object, Object> cache0 = ignite0.cache(CACHE_NAME);
+
+        BinaryObject bo = ignite0
+            .binary()
+            .builder(DYNAMIC_TYPE_NAME)
+            .setField(DYNAMIC_INT_FIELD_NAME, 10)
+            .build();
+
+        cache0.put(0, bo);
+
+        stopGrid("A");
+
+        IgniteCache<Object, Object> cache1 = ignite1.cache(CACHE_NAME);
+
+        bo = ignite1
+            .binary()
+            .builder(DYNAMIC_TYPE_NAME)
+            .setField(DYNAMIC_INT_FIELD_NAME, 20)
+            .setField(DYNAMIC_STR_FIELD_NAME, "str")
+            .build();
+
+        cache1.put(1, bo);
+
+        stopAllGrids();
+
+        ignite0 = startGridInASeparateWorkDir("A");
+        ignite1 = startGridInASeparateWorkDir("B");
+
+        awaitPartitionMapExchange();
+
+        cache0 = ignite0.cache(CACHE_NAME).withKeepBinary();
+
+        BinaryObject bObj0 = (BinaryObject) cache0.get(0);
+
+        assertEquals(10, (int) bObj0.field("intField"));
+
+        cache1 = ignite1.cache(CACHE_NAME).withKeepBinary();
+
+        BinaryObject bObj1 = (BinaryObject) cache1.get(1);
+
+        assertEquals(20, (int) bObj1.field("intField"));
+        assertEquals("str", bObj1.field("strField"));
+    }
+
+    /**
+     * @see <a href="https://issues.apache.org/jira/browse/IGNITE-7258">IGNITE-7258</a> refer to the following JIRA for more context about the problem verified by the test.
+     */
+    public void testNewBinaryMetadataIsWrittenOnOldCoordinator() throws Exception {
+        Ignite ignite0 = startGridInASeparateWorkDir("A");
+        Ignite ignite1 = startGridInASeparateWorkDir("B");
+
+        ignite0.active(true);
+
+        IgniteCache<Object, Object> cache0 = ignite0.cache(CACHE_NAME);
+
+        BinaryObject bObj0 = ignite0.binary()
+            .builder("DynamicType0").setField("intField", 10).build();
+
+        cache0.put(0, bObj0);
+
+        stopGrid("A");
+
+        IgniteCache<Object, Object> cache1 = ignite1.cache(CACHE_NAME);
+
+        BinaryObject bObj1 = ignite1.binary()
+            .builder("DynamicType1").setField("strField", "str").build();
+
+        cache1.put(1, bObj1);
+
+        stopAllGrids();
+
+        ignite0 = startGridInASeparateWorkDir("A");
+
+        startGridInASeparateWorkDir("B");
+
+        awaitPartitionMapExchange();
+
+        stopGrid("B");
+
+        cache0 = ignite0.cache(CACHE_NAME).withKeepBinary();
+
+        bObj0 = (BinaryObject) cache0.get(0);
+        bObj1 = (BinaryObject) cache0.get(1);
+
+        assertEquals("DynamicType0", binaryTypeName(bObj0));
+        assertEquals("DynamicType1", binaryTypeName(bObj1));
+
+        assertEquals(10, (int) bObj0.field(DYNAMIC_INT_FIELD_NAME));
+        assertEquals("str", bObj1.field(DYNAMIC_STR_FIELD_NAME));
+    }
+
+    /**
+     * Verifies that newer BinaryMetadata is applied not only on coordinator but on all nodes
+     * with outdated version.
+     *
+     * In the following test nodeB was offline when BinaryMetadata was updated,
+     * after full cluster restart it starts second (so it doesn't take the role of coordinator)
+     * but metadata update is propagated to it anyway.
+     *
+     * @see <a href="https://issues.apache.org/jira/browse/IGNITE-7258">IGNITE-7258</a> refer to the following JIRA for more context about the problem verified by the test.
+     */
+    public void testNewBinaryMetadataIsPropagatedToAllOutOfDataNodes() throws Exception {
+        Ignite igniteA = startGridInASeparateWorkDir("A");
+        startGridInASeparateWorkDir("B");
+        Ignite igniteC = startGridInASeparateWorkDir("C");
+        startGridInASeparateWorkDir("D");
+
+        igniteA.active(true);
+
+        BinaryObject bObj0 = igniteA.binary()
+            .builder(DYNAMIC_TYPE_NAME).setField(DYNAMIC_INT_FIELD_NAME, 10).build();
+
+        igniteA.cache(CACHE_NAME).put(0, bObj0);
+
+        stopGrid("A");
+        stopGrid("B");
+
+        BinaryObject bObj1 = igniteC.binary()
+            .builder(DYNAMIC_TYPE_NAME)
+            .setField(DYNAMIC_INT_FIELD_NAME, 20)
+            .setField(DYNAMIC_STR_FIELD_NAME, "str").build();
+
+        igniteC.cache(CACHE_NAME).put(1, bObj1);
+
+        //full cluster restart
+        stopAllGrids();
+
+        //node A becomes a coordinator
+        igniteA = startGridInASeparateWorkDir("A");
+        //node B isn't a coordinator, but metadata update is propagated to if
+        startGridInASeparateWorkDir("B");
+        //these two nodes will provide an updated version of metadata when started
+        startGridInASeparateWorkDir("C");
+        startGridInASeparateWorkDir("D");
+
+        awaitPartitionMapExchange();
+
+        //stopping everything to make sure that nodeB will lose its in-memory BinaryMetadata cache
+        // and will have to reload it from FS when restarted later
+        stopAllGrids();
+
+        Ignite igniteB = startGridInASeparateWorkDir("B");
+
+        igniteB.active(true);
+
+        bObj1 = (BinaryObject) igniteB.cache(CACHE_NAME).withKeepBinary().get(1);
+
+        assertEquals(20, (int) bObj1.field(DYNAMIC_INT_FIELD_NAME));
+        assertEquals("str", bObj1.field(DYNAMIC_STR_FIELD_NAME));
+    }
+
+    /** */
+    private Ignite startGridInASeparateWorkDir(String nodeName) throws Exception {
+        customWorkSubDir = String.format(CUSTOM_WORK_DIR_NAME_PATTERN, nodeName);
+        return startGrid(nodeName);
+    }
+
+    /** */
+    private String binaryTypeName(BinaryObject bObj) {
+        return bObj.type().typeName();
+    }
+
+    /**
+     * If joining node has incompatible BinaryMetadata (e.g. when user manually copies binary_meta file),
+     * coordinator detects it and fails the node providing information about conflict.
+     *
+     * @see <a href="https://issues.apache.org/jira/browse/IGNITE-7258">IGNITE-7258</a> refer to the following JIRA for more context about the problem verified by the test.
+     */
+    public void testNodeWithIncompatibleMetadataIsProhibitedToJoinTheCluster() throws Exception {
+        final String decimalFieldName = "decField";
+
+        Ignite igniteA = startGridInASeparateWorkDir("A");
+        Ignite igniteB = startGridInASeparateWorkDir("B");
+
+        String bConsId = igniteB.cluster().localNode().consistentId().toString();
+
+        igniteA.active(true);
+
+        IgniteCache<Object, Object> cache = igniteA.cache(CACHE_NAME);
+
+        BinaryObject bObj = igniteA.binary()
+            .builder(DYNAMIC_TYPE_NAME).setField(decimalFieldName, 10).build();
+
+        cache.put(0, bObj);
+
+        stopAllGrids();
+
+        Ignite igniteC = startGridInASeparateWorkDir("C");
+        String cConsId = igniteC.cluster().localNode().consistentId().toString();
+        igniteC.active(true);
+
+        cache = igniteC.cache(CACHE_NAME);
+        bObj = igniteC.binary().builder(DYNAMIC_TYPE_NAME).setField(decimalFieldName, 10L).build();
+
+        cache.put(0, bObj);
+
+        stopAllGrids();
+
+        copyIncompatibleBinaryMetadata(
+            String.format(CUSTOM_WORK_DIR_NAME_PATTERN, "C"),
+            cConsId,
+            String.format(CUSTOM_WORK_DIR_NAME_PATTERN, "B"),
+            bConsId,
+            DYNAMIC_TYPE_NAME.toLowerCase().hashCode() + ".bin");
+
+        startGridInASeparateWorkDir("A");
+
+        boolean exceptedExceptionThrown = false;
+        try {
+            startGridInASeparateWorkDir("B");
+        }
+        catch (Exception e) {
+            if (e.getCause() != null && e.getCause().getCause() != null) {
+                if (e.getCause().getCause().getMessage().contains(
+                        String.format("[typeName=%s, fieldName=%s, fieldTypeName1=int, fieldTypeName2=long]",
+                            DYNAMIC_TYPE_NAME,
+                            decimalFieldName)
+                ))
+                    exceptedExceptionThrown = true;
+            }
+            else
+                throw e;
+        }
+
+        assertTrue(exceptedExceptionThrown);
+    }
+
+    /** */
+    private void copyIncompatibleBinaryMetadata(String fromWorkDir,
+        String fromConsId,
+        String toWorkDir,
+        String toConsId,
+        String fileName
+    ) throws Exception {
+        String workDir = U.defaultWorkDirectory();
+
+        Path fromFile = Paths.get(workDir, fromWorkDir, "binary_meta", "node00-" + fromConsId, fileName);
+        Path toFile = Paths.get(workDir, toWorkDir, "binary_meta", "node00-" + toConsId, fileName);
+
+        Files.copy(fromFile, toFile, StandardCopyOption.REPLACE_EXISTING);
+    }
+
+    /**
      * Test verifies that binary metadata from regular java classes is saved and restored correctly
      * on cluster restart.
      */
@@ -237,7 +503,7 @@ public class IgnitePdsBinaryMetadataOnClusterRestartTest extends GridCommonAbstr
 
         awaitPartitionMapExchange();
 
-        examineDynamicMetadata(4, contentExaminer0, contentExaminer1, structureExaminer1);
+        examineDynamicMetadata(2, contentExaminer0, contentExaminer1, structureExaminer1);
     }
 
     /**
@@ -350,6 +616,8 @@ public class IgnitePdsBinaryMetadataOnClusterRestartTest extends GridCommonAbstr
         stopAllGrids();
 
         cleanIgniteWorkDir();
+
+        customWorkSubDir = null;
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
index 8e43e93..bf20ee4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
@@ -686,7 +686,12 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
     private static class CoordinatorNodeFilter implements IgnitePredicate<ClusterNode> {
         /** {@inheritDoc} */
         @Override public boolean apply(ClusterNode node) {
-            return node.order() > 1;
+            try {
+                return node.order() > 1;
+            }
+            catch (UnsupportedOperationException e) {
+                return false;
+            }
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
index e9828f5..301c0bc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
@@ -56,6 +56,8 @@ public class IgnitePersistentStoreDataStructuresTest extends GridCommonAbstractT
 
         cfg.setDataStorageConfiguration(memCfg);
 
+        cfg.setAutoActivationEnabled(false);
+
         return cfg;
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAllBaselineNodesOnlineFullApiSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAllBaselineNodesOnlineFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAllBaselineNodesOnlineFullApiSelfTest.java
new file mode 100644
index 0000000..c6aed7d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAllBaselineNodesOnlineFullApiSelfTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteAllBaselineNodesOnlineFullApiSelfTest extends IgniteBaselineAbstractFullApiSelfTest {
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        initStoreStrategy();
+
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(gridCount());
+
+        grid(0).active(true);
+
+        awaitPartitionMapExchange();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java
new file mode 100644
index 0000000..8dcfc0b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest;
+
+/**
+ *
+ */
+public abstract class IgniteBaselineAbstractFullApiSelfTest extends GridCacheAbstractFullApiSelfTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setMaxSize(200 * 1024 * 1024)
+                    .setPersistenceEnabled(true)));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected int gridCount() {
+        return 4;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getPartitionMapExchangeTimeout() {
+        return 60_000;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCachePutAllFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCachePutAllFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCachePutAllFailoverTest.java
new file mode 100644
index 0000000..16fa971
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCachePutAllFailoverTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.distributed.CachePutAllFailoverAbstractTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Failover test for cache putAll operations when BaselineTopology is changed down
+ * (existing node gets stopped and removed from BaselineTopology).
+ */
+public class IgniteChangingBaselineDownCachePutAllFailoverTest extends CachePutAllFailoverAbstractTest {
+    /** */
+    private static final int GRIDS_COUNT = 5;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                        .setInitialSize(200 * 1024 * 1024)
+                        .setMaxSize(200 * 1024 * 1024)
+                        .setCheckpointPageBufferSize(200 * 1024 * 1024)
+                )
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(GRIDS_COUNT);
+
+        grid(0).active(true);
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean finished, final long endTime) {
+        return GridTestUtils.runAsync(new Callable() {
+            @Override public Object call() throws Exception {
+                Thread.currentThread().setName("restart-thread");
+
+                U.sleep(15_000);
+
+                ThreadLocalRandom tlr = ThreadLocalRandom.current();
+
+                int idx = tlr.nextInt(1, GRIDS_COUNT);
+
+                log.info("Stopping node " + idx);
+
+                stopGrid(idx);
+
+                IgniteEx ig0 = grid(0);
+
+                ig0.cluster().setBaselineTopology(baselineNodes(ig0.cluster().forServers().nodes()));
+
+                U.sleep(3_000);
+
+                return null;
+            }
+        });
+    }
+
+    /** */
+    private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+        Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+        for (ClusterNode clN : clNodes)
+            res.add(clN);
+
+        return res;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCacheRemoveFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCacheRemoveFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCacheRemoveFailoverTest.java
new file mode 100644
index 0000000..86b5809
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCacheRemoveFailoverTest.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractRemoveFailureTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
+/**
+ * Failover test for cache remove operations when BaselineTopology is changed down
+ * (existing node gets stopped and removed from BaselineTopology).
+ */
+public class IgniteChangingBaselineDownCacheRemoveFailoverTest extends GridCacheAbstractRemoveFailureTest {
+    /** */
+    private static final int GRIDS_COUNT = 5;
+
+    /** {@inheritDoc} */
+    @Override protected CacheMode cacheMode() {
+        return PARTITIONED;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected CacheAtomicityMode atomicityMode() {
+        return TRANSACTIONAL;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected NearCacheConfiguration nearCache() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                    .setPersistenceEnabled(true)
+                    .setInitialSize(200 * 1024 * 1024)
+                    .setMaxSize(200 * 1024 * 1024)
+                    .setCheckpointPageBufferSize(200 * 1024 * 1024)
+                )
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(GRIDS_COUNT);
+
+        startGrid(GRIDS_COUNT);
+
+        grid(0).active(true);
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean stop, final AtomicReference<CyclicBarrier> cmp) {
+        return GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                Thread.currentThread().setName("restart-thread");
+
+                U.sleep(random(5000, 10_000));
+
+                log.info("Stopping node " + GRIDS_COUNT);
+
+                stopGrid(GRIDS_COUNT);
+
+                IgniteEx ig0 = grid(0);
+
+                ig0.cluster().setBaselineTopology(baselineNodes(ig0.cluster().forServers().nodes()));
+
+                while (!stop.get()) {
+                    CyclicBarrier barrier = cmp.get();
+
+                    if (barrier != null) {
+                        log.info("Wait data check.");
+
+                        barrier.await(60_000, TimeUnit.MILLISECONDS);
+
+                        log.info("Finished wait data check.");
+                    }
+                }
+
+                return null;
+            }
+        });
+    }
+
+    /** */
+    private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+        Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+        for (ClusterNode clN : clNodes)
+            res.add(clN);
+
+        return res;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCachePutAllFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCachePutAllFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCachePutAllFailoverTest.java
new file mode 100644
index 0000000..44c8402
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCachePutAllFailoverTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.distributed.CachePutAllFailoverAbstractTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Failover test for cache putAll operations when BaselineTopology is changed up
+ * (new node is added to BaselineTopology).
+ */
+public class IgniteChangingBaselineUpCachePutAllFailoverTest extends CachePutAllFailoverAbstractTest {
+    /** */
+    private static final int GRIDS_COUNT = 5;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                        .setInitialSize(200 * 1024 * 1024)
+                        .setMaxSize(200 * 1024 * 1024)
+                        .setCheckpointPageBufferSize(200 * 1024 * 1024)
+                )
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(GRIDS_COUNT);
+
+        grid(0).active(true);
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean finished, final long endTime) {
+        return GridTestUtils.runAsync(new Callable() {
+            @Override public Object call() throws Exception {
+                Thread.currentThread().setName("restart-thread");
+
+                U.sleep(15_000);
+
+                log.info("Starting node");
+
+                startGrid(GRIDS_COUNT);
+
+                IgniteEx ig0 = grid(0);
+
+                ig0.cluster().setBaselineTopology(baselineNodes(ig0.cluster().forServers().nodes()));
+
+                U.sleep(3_000);
+
+                return null;
+            }
+        });
+    }
+
+    /** */
+    private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+        Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+        for (ClusterNode clN : clNodes)
+            res.add(clN);
+
+        return res;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCacheRemoveFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCacheRemoveFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCacheRemoveFailoverTest.java
new file mode 100644
index 0000000..8391e0c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCacheRemoveFailoverTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractRemoveFailureTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
+/**
+ * Failover test for cache remove operations when BaselineTopology is changed up
+ * (new node is added to BaselineTopology).
+ */
+public class IgniteChangingBaselineUpCacheRemoveFailoverTest extends GridCacheAbstractRemoveFailureTest {
+    /** */
+    private static final int GRIDS_COUNT = 5;
+
+    /** {@inheritDoc} */
+    @Override protected CacheMode cacheMode() {
+        return PARTITIONED;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected CacheAtomicityMode atomicityMode() {
+        return TRANSACTIONAL;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected NearCacheConfiguration nearCache() {
+        return null;
+    }
+
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                    .setPersistenceEnabled(true)
+                    .setInitialSize(200 * 1024 * 1024)
+                    .setMaxSize(200 * 1024 * 1024)
+                    .setCheckpointPageBufferSize(200 * 1024 * 1024)
+                )
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(GRIDS_COUNT);
+
+        grid(0).active(true);
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean stop, final AtomicReference<CyclicBarrier> cmp) {
+        return GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                Thread.currentThread().setName("restart-thread");
+
+                U.sleep(random(5000, 10_000));
+
+                log.info("Starting node " + GRIDS_COUNT);
+
+                startGrid(GRIDS_COUNT);
+
+                IgniteEx ig0 = grid(0);
+
+                ig0.cluster().setBaselineTopology(baselineNodes(ig0.cluster().forServers().nodes()));
+
+                while (!stop.get()) {
+                    CyclicBarrier barrier = cmp.get();
+
+                    if (barrier != null) {
+                        log.info("Wait data check.");
+
+                        barrier.await(60_000, TimeUnit.MILLISECONDS);
+
+                        log.info("Finished wait data check.");
+                    }
+                }
+
+                return null;
+            }
+        });
+    }
+
+    /** */
+    private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+        Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+        for (ClusterNode clN : clNodes)
+            res.add(clN);
+
+        return res;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOfflineBaselineNodeFullApiSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOfflineBaselineNodeFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOfflineBaselineNodeFullApiSelfTest.java
new file mode 100644
index 0000000..3fd4974
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOfflineBaselineNodeFullApiSelfTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteOfflineBaselineNodeFullApiSelfTest extends IgniteBaselineAbstractFullApiSelfTest {
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        initStoreStrategy();
+
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(gridCount());
+
+        startGrid("offlineBaselineNode");
+
+        grid(0).active(true);
+
+        stopGrid("offlineBaselineNode");
+
+        awaitPartitionMapExchange();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOnlineNodeOutOfBaselineFullApiSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOnlineNodeOutOfBaselineFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOnlineNodeOutOfBaselineFullApiSelfTest.java
new file mode 100644
index 0000000..07c475b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOnlineNodeOutOfBaselineFullApiSelfTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteOnlineNodeOutOfBaselineFullApiSelfTest extends IgniteBaselineAbstractFullApiSelfTest {
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        initStoreStrategy();
+
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(gridCount());
+
+        grid(0).active(true);
+
+        startGrid("onlineOutOfBaselineNode");
+
+        awaitPartitionMapExchange();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCachePutAllFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCachePutAllFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCachePutAllFailoverTest.java
new file mode 100644
index 0000000..13f3489
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCachePutAllFailoverTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.distributed.CachePutAllFailoverAbstractTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteStableBaselineCachePutAllFailoverTest extends CachePutAllFailoverAbstractTest {
+    /** */
+    private static final int GRIDS_COUNT = 3;
+
+    /** */
+    private static final int OUT_OF_BASELINE_GRID_ID = GRIDS_COUNT;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                        .setInitialSize(200 * 1024 * 1024)
+                        .setMaxSize(200 * 1024 * 1024)
+                    .setCheckpointPageBufferSize(200 * 1024 * 1024)
+                )
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean finished, final long endTime) {
+        return GridTestUtils.runAsync(new Callable() {
+            @Override public Object call() throws Exception {
+                Thread.currentThread().setName("restart-thread");
+
+                while (!finished.get() && System.currentTimeMillis() < endTime) {
+                    ThreadLocalRandom tlr = ThreadLocalRandom.current();
+
+                    int idx = tlr.nextInt(1, GRIDS_COUNT + 1);
+
+                    log.info("Stopping node " + idx);
+
+                    stopGrid(idx);
+
+                    U.sleep(tlr.nextInt(500, 700));
+
+                    log.info("Restarting node " + idx);
+
+                    startGrid(idx);
+                }
+
+                return null;
+            }
+        });
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(GRIDS_COUNT);
+
+        grid(0).active(true);
+
+        startGrid(OUT_OF_BASELINE_GRID_ID);
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+}


[14/17] ignite git commit: IEP-4 Baseline topology for persistent caches (Phase 1) Contributed by: Dmitriy Govorukhin Dmitry Pavlov Eduard Shangareev Ily

Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index ed4c520..2a374a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import javax.cache.CacheException;
@@ -157,7 +158,8 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
             if (ctx.clientNode())
                 ctx.event().addLocalEventListener(clientDisconLsnr, EVT_CLIENT_NODE_DISCONNECTED);
 
-            metadataFileStore = new BinaryMetadataFileStore(metadataLocCache, ctx, log, binaryMetadataFileStoreDir);
+            if (!ctx.clientNode())
+                metadataFileStore = new BinaryMetadataFileStore(metadataLocCache, ctx, log, binaryMetadataFileStoreDir);
 
             transport = new BinaryMetadataTransport(metadataLocCache, metadataFileStore, ctx, log);
 
@@ -241,7 +243,8 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
                 }
             }
 
-            metadataFileStore.restoreMetadata();
+            if (!ctx.clientNode())
+                metadataFileStore.restoreMetadata();
         }
     }
 
@@ -865,15 +868,20 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode rmtNode) {
-        IgniteNodeValidationResult res = super.validateNode(rmtNode);
-
-        if (res != null)
-            return res;
+    @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode rmtNode, DiscoveryDataBag.JoiningNodeDiscoveryData discoData) {
+        IgniteNodeValidationResult res;
 
         if (getBoolean(IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK) || !(marsh instanceof BinaryMarshaller))
             return null;
 
+        if ((res = validateBinaryConfiguration(rmtNode)) != null)
+            return res;
+
+        return validateBinaryMetadata(rmtNode.id(), (Map<Integer, BinaryMetadataHolder>) discoData.joiningNodeData());
+    }
+
+    /** */
+    private IgniteNodeValidationResult validateBinaryConfiguration(ClusterNode rmtNode) {
         Object rmtBinaryCfg = rmtNode.attribute(IgniteNodeAttributes.ATTR_BINARY_CONFIGURATION);
 
         ClusterNode locNode = ctx.discovery().localNode();
@@ -892,6 +900,38 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
         return null;
     }
 
+    /** */
+    private IgniteNodeValidationResult validateBinaryMetadata(UUID rmtNodeId, Map<Integer, BinaryMetadataHolder> newNodeMeta) {
+        if (newNodeMeta == null)
+            return null;
+
+        for (Map.Entry<Integer, BinaryMetadataHolder> metaEntry : newNodeMeta.entrySet()) {
+            if (!metadataLocCache.containsKey(metaEntry.getKey()))
+                continue;
+
+            BinaryMetadata locMeta = metadataLocCache.get(metaEntry.getKey()).metadata();
+            BinaryMetadata rmtMeta = metaEntry.getValue().metadata();
+
+            if (locMeta == null || rmtMeta == null)
+                continue;
+
+            try {
+                BinaryUtils.mergeMetadata(locMeta, rmtMeta);
+            }
+            catch (Exception e) {
+                String locMsg = "Exception was thrown when merging binary metadata from node %s: %s";
+
+                String rmtMsg = "Exception was thrown on coordinator when merging binary metadata from this node: %s";
+
+                return new IgniteNodeValidationResult(rmtNodeId,
+                    String.format(locMsg, rmtNodeId.toString(), e.getMessage()),
+                    String.format(rmtMsg, e.getMessage()));
+            }
+        }
+
+        return null;
+    }
+
     /** {@inheritDoc} */
     @Nullable @Override public DiscoveryDataExchangeType discoveryDataType() {
         return BINARY_PROC;
@@ -910,6 +950,69 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
     }
 
     /** {@inheritDoc} */
+    @Override public void collectJoiningNodeData(DiscoveryDataBag dataBag) {
+        Map<Integer, BinaryMetadataHolder> res = U.newHashMap(metadataLocCache.size());
+
+        for (Map.Entry<Integer,BinaryMetadataHolder> e : metadataLocCache.entrySet())
+            res.put(e.getKey(), e.getValue());
+
+        dataBag.addJoiningNodeData(BINARY_PROC.ordinal(), (Serializable) res);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onJoiningNodeDataReceived(DiscoveryDataBag.JoiningNodeDiscoveryData data) {
+        Map<Integer,BinaryMetadataHolder> newNodeMeta = (Map<Integer, BinaryMetadataHolder>) data.joiningNodeData();
+
+        if (newNodeMeta == null)
+            return;
+
+        UUID joiningNode = data.joiningNodeId();
+
+        for (Map.Entry<Integer, BinaryMetadataHolder> metaEntry : newNodeMeta.entrySet()) {
+            if (metadataLocCache.containsKey(metaEntry.getKey())) {
+                BinaryMetadataHolder localMetaHolder = metadataLocCache.get(metaEntry.getKey());
+
+                BinaryMetadata newMeta = metaEntry.getValue().metadata();
+                BinaryMetadata localMeta = localMetaHolder.metadata();
+
+                BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(localMeta, newMeta);
+
+                if (mergedMeta != localMeta) {
+                    //put mergedMeta to local cache and store to disk
+                    U.log(log,
+                        String.format("Newer version of existing BinaryMetadata[typeId=%d, typeName=%s] " +
+                                "is received from node %s; updating it locally",
+                            mergedMeta.typeId(),
+                            mergedMeta.typeName(),
+                            joiningNode));
+
+                    metadataLocCache.put(metaEntry.getKey(),
+                        new BinaryMetadataHolder(mergedMeta,
+                            localMetaHolder.pendingVersion(),
+                            localMetaHolder.acceptedVersion()));
+
+                    metadataFileStore.writeMetadata(mergedMeta);
+                }
+            }
+            else {
+                BinaryMetadataHolder newMetaHolder = metaEntry.getValue();
+                BinaryMetadata newMeta = newMetaHolder.metadata();
+
+                U.log(log,
+                    String.format("New BinaryMetadata[typeId=%d, typeName=%s] " +
+                            "is received from node %s; adding it locally",
+                        newMeta.typeId(),
+                        newMeta.typeName(),
+                        joiningNode));
+
+                metadataLocCache.put(metaEntry.getKey(), newMetaHolder);
+
+                metadataFileStore.writeMetadata(newMeta);
+            }
+        }
+    }
+
+    /** {@inheritDoc} */
     @Override public void onGridDataReceived(GridDiscoveryData data) {
         Map<Integer, BinaryMetadataHolder> receivedData = (Map<Integer, BinaryMetadataHolder>) data.commonData();
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
index f84fb99..5733039 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
@@ -45,6 +45,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheOperation;
 import org.apache.ignite.internal.processors.cache.GridCacheReturn;
 import org.apache.ignite.internal.processors.cache.GridCacheReturnCompletableWrapper;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.GridCacheUpdateTxResult;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheEntry;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
@@ -480,7 +481,7 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
                     cctx.database().checkpointReadLock();
 
                     try {
-                        Collection<IgniteTxEntry> entries = near() ? allEntries() : writeEntries();
+                        Collection<IgniteTxEntry> entries = near() || cctx.snapshot().needTxReadLogging() ? allEntries() : writeEntries();
 
                         List<DataEntry> dataEntries = null;
 
@@ -563,7 +564,7 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
                                             GridCacheVersion dhtVer = cached.isNear() ? writeVersion() : null;
 
                                             if (!near() && cacheCtx.group().persistenceEnabled() &&
-                                                op != NOOP && op != RELOAD && op != READ) {
+                                                op != NOOP && op != RELOAD && (op != READ || cctx.snapshot().needTxReadLogging())) {
                                                 if (dataEntries == null)
                                                     dataEntries = new ArrayList<>(entries.size());
 
@@ -605,7 +606,7 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
                                                 else {
                                                     assert val != null : txEntry;
 
-                                                    cached.innerSet(this,
+                                                    GridCacheUpdateTxResult updRes = cached.innerSet(this,
                                                         eventNodeId(),
                                                         nodeId,
                                                         val,
@@ -627,6 +628,9 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
                                                         dhtVer,
                                                         txEntry.updateCounter());
 
+                                                    if (updRes.loggedPointer() != null)
+                                                        ptr = updRes.loggedPointer();
+
                                                     // Keep near entry up to date.
                                                     if (nearCached != null) {
                                                         CacheObject val0 = cached.valueBytes();
@@ -641,7 +645,7 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
                                                 }
                                             }
                                             else if (op == DELETE) {
-                                                cached.innerRemove(this,
+                                                GridCacheUpdateTxResult updRes = cached.innerRemove(this,
                                                     eventNodeId(),
                                                     nodeId,
                                                     false,
@@ -659,6 +663,9 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
                                                     dhtVer,
                                                     txEntry.updateCounter());
 
+                                                if (updRes.loggedPointer() != null)
+                                                    ptr = updRes.loggedPointer();
+
                                                 // Keep near entry up to date.
                                                 if (nearCached != null)
                                                     nearCached.updateOrEvict(xidVer, null, 0, 0, nodeId, topVer);
@@ -751,7 +758,7 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
                             if (!near() && !F.isEmpty(dataEntries) && cctx.wal() != null)
                                 cctx.wal().log(new DataRecord(dataEntries));
 
-                            if (ptr != null)
+                            if (ptr != null && !cctx.tm().logTxRecords())
                                 cctx.wal().fsync(ptr);
                         }
                         catch (StorageException e) {

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
index dcc08d4..f793328 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
@@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.internal.GridNodeOrderComparator;
+import org.apache.ignite.internal.cluster.NodeOrderComparator;
 import org.apache.ignite.internal.IgniteNeedReconnectException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
@@ -105,7 +105,7 @@ public class GridDhtAssignmentFetchFuture extends GridFutureAdapter<GridDhtAffin
                 tmp.add(node);
         }
 
-        Collections.sort(tmp, GridNodeOrderComparator.INSTANCE);
+        Collections.sort(tmp, NodeOrderComparator.getInstance());
 
         this.availableNodes = tmp;
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index 28f9c76..953b47d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -647,6 +647,8 @@ public abstract class GridDhtCacheAdapter<K, V> extends GridDistributedCacheAdap
             if (part.reserve()) {
                 GridCacheEntryEx entry = null;
 
+                ctx.shared().database().checkpointReadLock();
+
                 try {
                     long ttl = CU.ttlForLoad(plc);
 
@@ -678,6 +680,8 @@ public abstract class GridDhtCacheAdapter<K, V> extends GridDistributedCacheAdap
                         entry.context().evicts().touch(entry, topVer);
 
                     part.release();
+
+                    ctx.shared().database().checkpointReadUnlock();
                 }
             }
             else if (log.isDebugEnabled())

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 62e2b12..cd5a951 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -213,6 +213,9 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
             // TODO ignite-db
             throw new IgniteException(e);
         }
+
+        // Todo log moving state
+        casState(state.get(), MOVING);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index e09d611..9ccaecf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht;
 
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -38,6 +39,7 @@ import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
 import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
@@ -49,6 +51,8 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.Gri
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.util.F0;
 import org.apache.ignite.internal.util.GridAtomicLong;
 import org.apache.ignite.internal.util.GridPartitionStateMap;
@@ -240,7 +244,8 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
         try {
             AffinityTopologyVersion exchTopVer = exchFut.initialVersion();
 
-            assert exchTopVer.compareTo(readyTopVer) > 0 : "Invalid topology version [topVer=" + readyTopVer +
+            assert exchTopVer.compareTo(readyTopVer) > 0 : "Invalid topology version [grp=" + grp.cacheOrGroupName() +
+                ", topVer=" + readyTopVer +
                 ", exchTopVer=" + exchTopVer +
                 ", fut=" + exchFut + ']';
 
@@ -354,16 +359,27 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
                     assert exchId.isJoined() || added;
 
                     for (int p = 0; p < num; p++) {
-                        if (localNode(p, aff)) {
+                        IgnitePageStoreManager storeMgr = ctx.pageStore();
+
+                        if (localNode(p, aff)
+                            || (storeMgr instanceof FilePageStoreManager
+                            && grp.persistenceEnabled()
+                            && Files.exists(((FilePageStoreManager)storeMgr).getPath(grp.sharedGroup(), grp.cacheOrGroupName(), p)))) {
                             GridDhtLocalPartition locPart = createPartition(p);
 
-                            boolean owned = locPart.own();
+                            if (grp.persistenceEnabled()) {
+                                GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)grp.shared().database();
 
-                            assert owned : "Failed to own partition for oldest node [grp=" + grp.cacheOrGroupName() +
-                                ", part=" + locPart + ']';
+                                locPart.restoreState(db.readPartitionState(grp, locPart.id()));
+                            }
+                            else {
+                                boolean owned = locPart.own();
 
-                            if (log.isDebugEnabled()) {
-                                log.debug("Owned partition for oldest node [grp=" + grp.cacheOrGroupName() +
+                                assert owned : "Failed to own partition for oldest node [grp=" + grp.cacheOrGroupName() +
+                                    ", part=" + locPart + ']';
+
+                                if (log.isDebugEnabled())
+                                    log.debug("Owned partition for oldest node [grp=" + grp.cacheOrGroupName() +
                                     ", part=" + locPart + ']');
                             }
 
@@ -782,7 +798,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
         try {
             GridDhtLocalPartition part = locParts.get(p);
 
-            if (part != null)
+            if (part != null && part.state() != EVICTED)
                 return part;
 
             part = new GridDhtLocalPartition(ctx, grp, p);
@@ -1109,10 +1125,12 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
      * @param states Additional partition states.
      * @return List of nodes for the partition.
      */
-    private List<ClusterNode> nodes(int p,
+    private List<ClusterNode> nodes(
+        int p,
         AffinityTopologyVersion topVer,
         GridDhtPartitionState state,
-        GridDhtPartitionState... states) {
+        GridDhtPartitionState... states
+    ) {
         Collection<UUID> allIds = F.nodeIds(discoCache.cacheGroupAffinityNodes(grp.groupId()));
 
         lock.readLock().lock();
@@ -1534,13 +1552,26 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
                 if (part == null)
                     continue;
 
-                long updCntr = cntrMap.updateCounter(part.id());
+                boolean reserve = part.reserve();
+
+                try {
+                    GridDhtPartitionState state = part.state();
+
+                    if (!reserve || state == EVICTED || state == RENTING)
+                        continue;
+
+                    long updCntr = cntrMap.updateCounter(part.id());
 
-                if (updCntr > part.updateCounter())
-                    part.updateCounter(updCntr);
-                else if (part.updateCounter() > 0) {
-                    cntrMap.initialUpdateCounter(part.id(), part.initialUpdateCounter());
-                    cntrMap.updateCounter(part.id(), part.updateCounter());
+                    if (updCntr > part.updateCounter())
+                        part.updateCounter(updCntr);
+                    else if (part.updateCounter() > 0) {
+                        cntrMap.initialUpdateCounter(part.id(), part.initialUpdateCounter());
+                        cntrMap.updateCounter(part.id(), part.updateCounter());
+                    }
+                }
+                finally {
+                    if (reserve)
+                        part.release();
                 }
             }
         }
@@ -1868,17 +1899,6 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
                         if (locPart != null) {
                             boolean marked = plc == PartitionLossPolicy.IGNORE ? locPart.own() : locPart.markLost();
 
-                            if (!marked && locPart.state() == RENTING)
-                                try {
-                                    //TODO https://issues.apache.org/jira/browse/IGNITE-6433
-                                    locPart.tryEvict();
-                                    locPart.rent(false).get();
-                                }
-                                catch (IgniteCheckedException e) {
-                                    U.error(log, "Failed to wait for RENTING partition eviction after partition LOST event",
-                                        e);
-                                }
-
                             if (marked)
                                 updateLocal(locPart.id(), locPart.state(), updSeq, resTopVer);
 
@@ -2067,6 +2087,9 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
      * @return Checks if any of the local partitions need to be evicted.
      */
     private boolean checkEvictions(long updateSeq, AffinityAssignment aff) {
+        if (!ctx.kernalContext().state().evictionsAllowed())
+            return false;
+
         boolean changed = false;
 
         UUID locId = ctx.localNodeId();
@@ -2105,7 +2128,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
                         int ownerCnt = nodeIds.size();
                         int affCnt = affNodes.size();
 
-                        if (ownerCnt > affCnt) {
+                        if (ownerCnt > affCnt) { //TODO !!! we could loss all owners in such case. Should be fixed by GG-13223
                             // Sort by node orders in ascending order.
                             Collections.sort(nodes, CU.nodeComparator(true));
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java
index d04870a..3c3150a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java
@@ -80,7 +80,7 @@ public abstract class GridDhtTopologyFutureAdapter extends GridFutureAdapter<Aff
         if (err != null)
             return err;
 
-        if (!cctx.shared().kernalContext().state().publicApiActiveState())
+        if (!cctx.shared().kernalContext().state().publicApiActiveState(true))
             return new CacheInvalidStateException(
                 "Failed to perform cache operation (cluster is not activated): " + cctx.name());
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
index e4a7141..c7282a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
@@ -474,13 +474,7 @@ public abstract class GridDhtTxLocalAdapter extends IgniteTxLocalAdapter {
             IgniteTxEntry existing = entry(e.txKey());
 
             if (existing != null) {
-                // Must keep NOOP operation if received READ because it means that the lock was sent to a backup node.
-                if (e.op() == READ) {
-                    if (existing.op() != NOOP)
-                        existing.op(e.op());
-                }
-                else
-                    existing.op(e.op()); // Absolutely must set operation, as default is DELETE.
+                existing.op(e.op()); // Absolutely must set operation, as default is DELETE.
 
                 existing.value(e.value(), e.hasWriteValue(), e.hasReadValue());
                 existing.entryProcessors(e.entryProcessors());

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
index a1f3984..bde15c2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
@@ -347,6 +347,8 @@ public final class GridDhtTxPrepareFuture extends GridCacheCompoundFuture<Ignite
 
             ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry);
 
+            cctx.database().checkpointReadLock();
+
             try {
                 if ((txEntry.op() == CREATE || txEntry.op() == UPDATE) &&
                     txEntry.conflictExpireTime() == CU.EXPIRE_TIME_CALCULATE) {
@@ -509,6 +511,9 @@ public final class GridDhtTxPrepareFuture extends GridCacheCompoundFuture<Ignite
             catch (GridCacheEntryRemovedException e) {
                 assert false : "Got entry removed exception while holding transactional lock on entry [e=" + e + ", cached=" + cached + ']';
             }
+            finally {
+                cctx.database().checkpointReadUnlock();
+            }
         }
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index 87d615e..179a8ae 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -395,10 +395,17 @@ public class GridPartitionedGetFuture<K, V> extends CacheDistributedGetFutureAda
         }
 
         boolean fastLocGet = (!forcePrimary || affNodes.get(0).isLocal()) &&
-            cctx.allowFastLocalRead(part, affNodes, topVer);
+            cctx.reserveForFastLocalGet(part, topVer);
 
-        if (fastLocGet && localGet(topVer, key, part, locVals))
-            return false;
+        if (fastLocGet) {
+            try {
+                if (localGet(topVer, key, part, locVals))
+                    return false;
+            }
+            finally {
+                cctx.releaseForFastLocalGet(part, topVer);
+            }
+        }
 
         ClusterNode node = affinityNode(affNodes);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index adb634a..4e34bcb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -335,10 +335,17 @@ public class GridPartitionedSingleGetFuture extends GridCacheFutureAdapter<Objec
         }
 
         boolean fastLocGet = (!forcePrimary || affNodes.get(0).isLocal()) &&
-            cctx.allowFastLocalRead(part, affNodes, topVer);
+            cctx.reserveForFastLocalGet(part, topVer);
 
-        if (fastLocGet && localGet(topVer, part))
-            return null;
+        if (fastLocGet) {
+            try {
+                if (localGet(topVer, part))
+                    return null;
+            }
+            finally {
+                cctx.releaseForFastLocalGet(part, topVer);
+            }
+        }
 
         ClusterNode affNode = affinityNode(affNodes);
 
@@ -659,6 +666,8 @@ public class GridPartitionedSingleGetFuture extends GridCacheFutureAdapter<Objec
      * @param ver Version.
      */
     private void setResult(@Nullable CacheObject val, @Nullable GridCacheVersion ver) {
+        cctx.shared().database().checkpointReadLock();
+
         try {
             assert !skipVals;
 
@@ -680,6 +689,9 @@ public class GridPartitionedSingleGetFuture extends GridCacheFutureAdapter<Objec
         catch (Exception e) {
             onDone(e);
         }
+        finally {
+            cctx.shared().database().checkpointReadUnlock();
+        }
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java
index 4b9aef0..d6956a6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java
@@ -87,6 +87,7 @@ public class GridNearAtomicFullUpdateRequest extends GridNearAtomicAbstractUpdat
 
     /** Conflict expire times. */
     private GridLongList conflictExpireTimes;
+
     /** Optional arguments for entry processor. */
     @GridDirectTransient
     private Object[] invokeArgs;

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
index 0648b4e..3536908 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
@@ -18,12 +18,14 @@
 package org.apache.ignite.internal.processors.cache.distributed.dht.colocated;
 
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheOperation;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedCacheEntry;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.Nullable;
@@ -76,6 +78,12 @@ public class GridDhtDetachedCacheEntry extends GridDistributedCacheEntry {
     }
 
     /** {@inheritDoc} */
+    @Override protected WALPointer logTxUpdate(IgniteInternalTx tx, CacheObject val, long expireTime, long updCntr)
+        throws IgniteCheckedException {
+        return null;
+    }
+
+    /** {@inheritDoc} */
     @Override protected void removeValue() throws IgniteCheckedException {
         // No-op for detached entries, index is updated on primary or backup nodes.
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java
index 83c0231..c0de7cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java
@@ -181,4 +181,21 @@ public class CachePartitionPartialCountersMap implements Serializable {
 
         return map0;
     }
+
+    /** {@inheritDoc} */
+    public String toString() {
+        StringBuilder sb = new StringBuilder("CachePartitionPartialCountersMap {");
+
+        for (int i = 0; i < partIds.length; i++) {
+            sb.append(partIds[i]).append("=(").append(initialUpdCntrs[i]).append(",")
+                .append(updCntrs[i]).append(")");
+
+            if (i != partIds.length - 1)
+                sb.append(", ");
+        }
+
+        sb.append("}");
+
+        return sb.toString();
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index d6f15c9..0c2de93 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -272,7 +272,7 @@ public class GridDhtPartitionDemander {
 
         long delay = grp.config().getRebalanceDelay();
 
-        if (delay == 0 || force) {
+        if ((delay == 0 || force) && assigns != null) {
             final RebalanceFuture oldFut = rebalanceFut;
 
             final RebalanceFuture fut = new RebalanceFuture(grp, assigns, log, cnt);

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index a8b195d..cf2c925 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -55,6 +55,7 @@ import org.apache.ignite.internal.events.DiscoveryCustomEvent;
 import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.pagemem.wal.record.ExchangeRecord;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
 import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage;
@@ -69,6 +70,7 @@ import org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.LocalJoinCachesContext;
 import org.apache.ignite.internal.processors.cache.StateChangeRequest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
@@ -76,10 +78,13 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartit
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
 import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage;
 import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage;
+import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
 import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -107,6 +112,8 @@ import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
+import static org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents.serverJoinEvent;
+import static org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents.serverLeftEvent;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap.PARTIAL_COUNTERS_MAP_SINCE;
 
 /**
@@ -245,7 +252,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
 
     /** */
     @GridToStringExclude
-    private volatile IgniteDhtPartitionsToReloadMap partsToReload = new IgniteDhtPartitionsToReloadMap();
+    private final IgniteDhtPartitionsToReloadMap partsToReload = new IgniteDhtPartitionsToReloadMap();
 
     /** */
     private final AtomicBoolean done = new AtomicBoolean();
@@ -458,6 +465,11 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
         return exchActions != null && exchActions.deactivate();
     }
 
+    /** */
+    public boolean changedBaseline() {
+        return exchActions != null && exchActions.changedBaseline();
+    }
+
     /**
      * @return First event discovery event.
      *
@@ -706,8 +718,10 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
     private void initCachesOnLocalJoin() throws IgniteCheckedException {
         cctx.activate();
 
-        List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> caches =
-            cctx.cache().cachesToStartOnLocalJoin();
+        LocalJoinCachesContext locJoinCtx = exchActions == null ? null : exchActions.localJoinContext();
+
+        List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> caches = locJoinCtx == null ? null :
+            locJoinCtx.caches();
 
         if (!cctx.kernalContext().clientNode()) {
             List<DynamicCacheDescriptor> startDescs = new ArrayList<>();
@@ -724,7 +738,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
             cctx.database().readCheckpointAndRestoreMemory(startDescs);
         }
 
-        cctx.cache().startCachesOnLocalJoin(caches, initialVersion());
+        cctx.cache().startCachesOnLocalJoin(locJoinCtx, initialVersion());
     }
 
     /**
@@ -797,74 +811,99 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
 
         assert req != null : exchActions;
 
-        if (req.activate()) {
-            if (log.isInfoEnabled()) {
-                log.info("Start activation process [nodeId=" + cctx.localNodeId() +
-                    ", client=" + cctx.kernalContext().clientNode() +
-                    ", topVer=" + initialVersion() + "]");
-            }
+        DiscoveryDataClusterState state = cctx.kernalContext().state().clusterState();
 
-            try {
-                cctx.activate();
+        if (state.transitionError() != null)
+            changeGlobalStateE = state.transitionError();
 
-                if (!cctx.kernalContext().clientNode()) {
-                    List<DynamicCacheDescriptor> startDescs = new ArrayList<>();
+        if (req.activeChanged()) {
+            if (req.activate()) {
+                if (log.isInfoEnabled()) {
+                    log.info("Start activation process [nodeId=" + cctx.localNodeId() +
+                        ", client=" + cctx.kernalContext().clientNode() +
+                        ", topVer=" + initialVersion() + "]");
+                }
 
-                    for (ExchangeActions.CacheActionData startReq : exchActions.cacheStartRequests()) {
-                        DynamicCacheDescriptor desc = startReq.descriptor();
+                try {
+                    cctx.activate();
 
-                        if (CU.isPersistentCache(desc.cacheConfiguration(), cctx.gridConfig().getDataStorageConfiguration()))
-                            startDescs.add(desc);
+                    if (!cctx.kernalContext().clientNode()) {
+                        List<DynamicCacheDescriptor> startDescs = new ArrayList<>();
+
+                        for (ExchangeActions.CacheActionData startReq : exchActions.cacheStartRequests()) {
+                            DynamicCacheDescriptor desc = startReq.descriptor();
+
+                            if (CU.isPersistentCache(desc.cacheConfiguration(),
+                                cctx.gridConfig().getDataStorageConfiguration()))
+                                startDescs.add(desc);
+                        }
+
+                        cctx.database().readCheckpointAndRestoreMemory(startDescs);
                     }
 
-                    cctx.database().readCheckpointAndRestoreMemory(startDescs);
+                    cctx.affinity().onCacheChangeRequest(this, crd, exchActions);
+
+                    if (log.isInfoEnabled()) {
+                        log.info("Successfully activated caches [nodeId=" + cctx.localNodeId() +
+                            ", client=" + cctx.kernalContext().clientNode() +
+                            ", topVer=" + initialVersion() + "]");
+                    }
                 }
+                catch (Exception e) {
+                    U.error(log, "Failed to activate node components [nodeId=" + cctx.localNodeId() +
+                        ", client=" + cctx.kernalContext().clientNode() +
+                        ", topVer=" + initialVersion() + "]", e);
 
-                cctx.affinity().onCacheChangeRequest(this, crd, exchActions);
+                    changeGlobalStateE = e;
 
+                    if (crd) {
+                        synchronized (mux) {
+                            changeGlobalStateExceptions.put(cctx.localNodeId(), e);
+                        }
+                    }
+                }
+            }
+            else {
                 if (log.isInfoEnabled()) {
-                    log.info("Successfully activated caches [nodeId=" + cctx.localNodeId() +
+                    log.info("Start deactivation process [nodeId=" + cctx.localNodeId() +
                         ", client=" + cctx.kernalContext().clientNode() +
                         ", topVer=" + initialVersion() + "]");
                 }
-            }
-            catch (Exception e) {
-                U.error(log, "Failed to activate node components [nodeId=" + cctx.localNodeId() +
-                    ", client=" + cctx.kernalContext().clientNode() +
-                    ", topVer=" + initialVersion() + "]", e);
 
-                changeGlobalStateE = e;
+                try {
+                    cctx.kernalContext().dataStructures().onDeActivate(cctx.kernalContext());
+
+                    cctx.kernalContext().service().onDeActivate(cctx.kernalContext());
+
+                    cctx.affinity().onCacheChangeRequest(this, crd, exchActions);
 
-                if (crd) {
-                    synchronized (mux) {
-                        changeGlobalStateExceptions.put(cctx.localNodeId(), e);
+                    if (log.isInfoEnabled()) {
+                        log.info("Successfully deactivated data structures, services and caches [" +
+                            "nodeId=" + cctx.localNodeId() +
+                            ", client=" + cctx.kernalContext().clientNode() +
+                            ", topVer=" + initialVersion() + "]");
                     }
                 }
+                catch (Exception e) {
+                    U.error(log, "Failed to deactivate node components [nodeId=" + cctx.localNodeId() +
+                        ", client=" + cctx.kernalContext().clientNode() +
+                        ", topVer=" + initialVersion() + "]", e);
+
+                    changeGlobalStateE = e;
+                }
             }
         }
-        else {
-            if (log.isInfoEnabled()) {
-                log.info("Start deactivation process [nodeId=" + cctx.localNodeId() +
-                    ", client=" + cctx.kernalContext().clientNode() +
-                    ", topVer=" + initialVersion() + "]");
-            }
-
+        else if (req.activate()) {
+            // TODO: BLT changes on inactive cluster can't be handled easily because persistent storage hasn't been initialized yet.
             try {
-                cctx.kernalContext().dataStructures().onDeActivate(cctx.kernalContext());
+                cctx.affinity().onBaselineTopologyChanged(this, crd);
 
-                cctx.kernalContext().service().onDeActivate(cctx.kernalContext());
-
-                cctx.affinity().onCacheChangeRequest(this, crd, exchActions);
-
-                if (log.isInfoEnabled()) {
-                    log.info("Successfully deactivated data structures, services and caches [" +
-                        "nodeId=" + cctx.localNodeId() +
-                        ", client=" + cctx.kernalContext().clientNode() +
-                        ", topVer=" + initialVersion() + "]");
-                }
+                if (CU.isPersistenceEnabled(cctx.kernalContext().config()) && !cctx.kernalContext().clientNode())
+                    cctx.kernalContext().state().onBaselineTopologyChanged(req.baselineTopology(),
+                        req.prevBaselineTopologyHistoryItem());
             }
             catch (Exception e) {
-                U.error(log, "Failed to deactivate node components [nodeId=" + cctx.localNodeId() +
+                U.error(log, "Failed to change baseline topology [nodeId=" + cctx.localNodeId() +
                     ", client=" + cctx.kernalContext().clientNode() +
                     ", topVer=" + initialVersion() + "]", e);
 
@@ -1021,6 +1060,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
             }
         }
 
+        cctx.database().beforeExchange(this);
+
         if (!exchCtx.mergeExchanges()) {
             for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
                 if (grp.isLocal() || cacheGroupStopping(grp.groupId()))
@@ -1032,8 +1073,6 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
             }
         }
 
-        cctx.database().beforeExchange(this);
-
         if (crd.isLocal()) {
             if (remaining.isEmpty())
                 onAllReceived(null);
@@ -1521,7 +1560,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
                 ExchangeDiscoveryEvents evts = exchCtx.events();
 
                 for (DiscoveryEvent evt : exchCtx.events().events()) {
-                    if (evts.serverLeftEvent(evt)) {
+                    if (serverLeftEvent(evt)) {
                         for (CacheGroupContext grp : cctx.cache().cacheGroups())
                             grp.affinityFunction().removeNode(evt.eventNode().id());
                     }
@@ -1533,9 +1572,20 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
             if (firstDiscoEvt instanceof DiscoveryCustomEvent)
                 ((DiscoveryCustomEvent)firstDiscoEvt).customMessage(null);
 
-            if (err == null)
+            if (err == null) {
                 cctx.exchange().lastFinishedFuture(this);
 
+                if (exchCtx != null && (exchCtx.events().hasServerLeft() || exchCtx.events().hasServerJoin())) {
+                    ExchangeDiscoveryEvents evts = exchCtx.events();
+
+                    for (DiscoveryEvent evt : exchCtx.events().events()) {
+                        if (serverLeftEvent(evt) || serverJoinEvent(evt))
+                            logExchange(evt);
+                    }
+                }
+
+            }
+
             return true;
         }
 
@@ -1543,6 +1593,37 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
     }
 
     /**
+     * Log exchange event.
+     *
+     * @param evt Discovery event.
+     */
+    private void logExchange(DiscoveryEvent evt) {
+        if (cctx.kernalContext().state().publicApiActiveState(false) && cctx.wal() != null) {
+            if (((FileWriteAheadLogManager)cctx.wal()).serializerVersion() > 1)
+                try {
+                    ExchangeRecord.Type type = null;
+
+                    if (evt.type() == EVT_NODE_JOINED)
+                        type = ExchangeRecord.Type.JOIN;
+                    else if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED)
+                        type = ExchangeRecord.Type.LEFT;
+
+                    BaselineTopology blt = cctx.kernalContext().state().clusterState().baselineTopology();
+
+                    if (type != null && blt != null) {
+                        Short constId = blt.consistentIdMapping().get(evt.eventNode().consistentId());
+
+                        if (constId != null)
+                            cctx.wal().log(new ExchangeRecord(constId, type));
+                    }
+                }
+                catch (IgniteCheckedException e) {
+                    U.error(log, "Fail during log exchange record.", e);
+                }
+        }
+    }
+
+    /**
      * Cleans up resources to avoid excessive memory usage.
      */
     public void cleanUp() {
@@ -2047,7 +2128,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
             if (state != GridDhtPartitionState.OWNING && state != GridDhtPartitionState.MOVING)
                 continue;
 
-            long cntr = state == GridDhtPartitionState.MOVING ? part.initialUpdateCounter() : part.updateCounter();
+            final long cntr = state == GridDhtPartitionState.MOVING ? part.initialUpdateCounter() : part.updateCounter();
 
             Long minCntr = minCntrs.get(part.id());
 
@@ -2301,25 +2382,27 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
                 }
             }
 
-            for (CacheGroupContext grpCtx : cctx.cache().cacheGroups()) {
-                if (!grpCtx.isLocal())
-                    grpCtx.topology().applyUpdateCounters();
-            }
-
             if (firstDiscoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT) {
                 assert firstDiscoEvt instanceof DiscoveryCustomEvent;
 
-                if (activateCluster())
+                if (activateCluster() || changedBaseline())
                     assignPartitionsStates();
 
-                if (((DiscoveryCustomEvent)firstDiscoEvt).customMessage() instanceof DynamicCacheChangeBatch) {
+                DiscoveryCustomMessage discoveryCustomMessage = ((DiscoveryCustomEvent) firstDiscoEvt).customMessage();
+
+                if (discoveryCustomMessage instanceof DynamicCacheChangeBatch) {
                     if (exchActions != null) {
+                        assignPartitionsStates();
+
                         Set<String> caches = exchActions.cachesToResetLostPartitions();
 
                         if (!F.isEmpty(caches))
                             resetLostPartitions(caches);
                     }
                 }
+                else if (discoveryCustomMessage instanceof SnapshotDiscoveryMessage
+                        && ((SnapshotDiscoveryMessage)discoveryCustomMessage).needAssignPartitions())
+                    assignPartitionsStates();
             }
             else {
                 if (exchCtx.events().hasServerJoin())
@@ -2329,6 +2412,11 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
                     detectLostPartitions(resTopVer);
             }
 
+            for (CacheGroupContext grpCtx : cctx.cache().cacheGroups()) {
+                if (!grpCtx.isLocal())
+                    grpCtx.topology().applyUpdateCounters();
+            }
+
             updateLastVersion(cctx.versions().last());
 
             cctx.versions().onExchange(lastVer.get().order());
@@ -2415,12 +2503,31 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
 
                         cctx.kernalContext().state().onStateChangeError(changeGlobalStateExceptions, req);
                     }
+                    else {
+                        boolean hasMoving = !partsToReload.isEmpty();
+
+                        Set<Integer> waitGrps = cctx.affinity().waitGroups();
+
+                        if (!hasMoving) {
+                            for (CacheGroupContext grpCtx : cctx.cache().cacheGroups()) {
+                                if (waitGrps.contains(grpCtx.groupId()) && grpCtx.topology().hasMovingPartitions()) {
+                                    hasMoving = true;
+
+                                    break;
+                                }
+
+                            }
+                        }
+
+                        cctx.kernalContext().state().onExchangeFinishedOnCoordinator(this, hasMoving);
+                    }
 
                     boolean active = !stateChangeErr && req.activate();
 
                     ChangeGlobalStateFinishMessage stateFinishMsg = new ChangeGlobalStateFinishMessage(
                         req.requestId(),
-                        active);
+                        active,
+                        !stateChangeErr);
 
                     cctx.discovery().sendCustomEvent(stateFinishMsg);
                 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java
index 2a72e95..37ca7e4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java
@@ -85,4 +85,11 @@ public class IgniteDhtPartitionsToReloadMap implements Serializable {
 
         parts.add(partId);
     }
+
+    /**
+     * @return {@code True} if empty.
+     */
+    public boolean isEmpty() {
+        return map == null || map.isEmpty();
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/StopCachesOnClientReconnectExchangeTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/StopCachesOnClientReconnectExchangeTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/StopCachesOnClientReconnectExchangeTask.java
new file mode 100644
index 0000000..e89bbd1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/StopCachesOnClientReconnectExchangeTask.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
+
+import java.util.Collection;
+import org.apache.ignite.internal.processors.cache.CachePartitionExchangeWorkerTask;
+import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ *
+ */
+public class StopCachesOnClientReconnectExchangeTask  extends GridFutureAdapter<Void>
+    implements CachePartitionExchangeWorkerTask {
+    /** */
+    @GridToStringInclude
+    private final Collection<GridCacheAdapter> stoppedCaches;
+
+    /**
+     * @param stoppedCaches Collection of stopped caches.
+     */
+    public StopCachesOnClientReconnectExchangeTask(Collection<GridCacheAdapter> stoppedCaches) {
+        this.stoppedCaches = stoppedCaches;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean skipForExchangeMerge() {
+        return false;
+    }
+
+    /**
+     * @return Collection of stopped caches.
+     */
+    public Collection<GridCacheAdapter> stoppedCaches() {
+        return stoppedCaches;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(StopCachesOnClientReconnectExchangeTask.class, this, super.toString());
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
index c4523ac..322e63c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.cache.distributed.near;
 import java.util.UUID;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheLockCandidates;
 import org.apache.ignite.internal.processors.cache.CacheObject;
@@ -473,6 +474,12 @@ public class GridNearCacheEntry extends GridDistributedCacheEntry {
     }
 
     /** {@inheritDoc} */
+    @Override protected WALPointer logTxUpdate(IgniteInternalTx tx, CacheObject val, long expireTime, long updCntr)
+        throws IgniteCheckedException {
+        return null;
+    }
+
+    /** {@inheritDoc} */
     @Nullable @Override public CacheDataRow unswap(CacheDataRow row, boolean checkExpire) {
         return null;
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index b175018..29dd12f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -478,10 +478,17 @@ public final class GridNearGetFuture<K, V> extends CacheDistributedGetFutureAdap
                 }
 
                 if (v == null) {
-                    boolean fastLocGet = allowLocRead && cctx.allowFastLocalRead(part, affNodes, topVer);
+                    boolean fastLocGet = allowLocRead && cctx.reserveForFastLocalGet(part, topVer);
 
-                    if (fastLocGet && localDhtGet(key, part, topVer, isNear))
-                        break;
+                    if (fastLocGet) {
+                        try {
+                            if (localDhtGet(key, part, topVer, isNear))
+                                break;
+                        }
+                        finally {
+                            cctx.releaseForFastLocalGet(part, topVer);
+                        }
+                    }
 
                     ClusterNode affNode = affinityNode(affNodes);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
index 6cf076b..8d8c0b2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
@@ -30,6 +30,8 @@ import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.cluster.ClusterTopologyException;
+import org.apache.ignite.internal.IgniteDiagnosticAware;
+import org.apache.ignite.internal.IgniteDiagnosticPrepareContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException;
@@ -45,6 +47,7 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.transactions.TxDeadlock;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
 import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
@@ -71,7 +74,8 @@ import static org.apache.ignite.transactions.TransactionState.PREPARING;
 /**
  *
  */
-public class GridNearOptimisticTxPrepareFuture extends GridNearOptimisticTxPrepareFutureAdapter {
+public class GridNearOptimisticTxPrepareFuture extends GridNearOptimisticTxPrepareFutureAdapter implements
+    IgniteDiagnosticAware {
     /** */
     @GridToStringExclude
     private KeyLockFuture keyLockFut;
@@ -783,6 +787,61 @@ public class GridNearOptimisticTxPrepareFuture extends GridNearOptimisticTxPrepa
     }
 
     /** {@inheritDoc} */
+    @Override public void addDiagnosticRequest(IgniteDiagnosticPrepareContext ctx) {
+        if (!isDone()) {
+            for (IgniteInternalFuture<GridNearTxPrepareResponse> fut : futures()) {
+                if (!fut.isDone()) {
+                    if (fut instanceof MiniFuture) {
+                        MiniFuture miniFut = (MiniFuture)fut;
+
+                        UUID nodeId = miniFut.node().id();
+                        GridCacheVersion dhtVer = miniFut.m.dhtVersion();
+                        GridCacheVersion nearVer = tx.nearXidVersion();
+
+                        if (dhtVer != null) {
+                            ctx.remoteTxInfo(
+                                nodeId,
+                                dhtVer,
+                                nearVer,
+                                "GridNearOptimisticTxPrepareFuture waiting for remote node response [" +
+                                    "nodeId=" + nodeId +
+                                    ", topVer=" + tx.topologyVersion() +
+                                    ", dhtVer=" + dhtVer +
+                                    ", nearVer=" + nearVer +
+                                    ", futId=" + futId +
+                                    ", miniId=" + miniFut.futId +
+                                    ", tx=" + tx + ']');
+                        }
+                        else {
+                            ctx.basicInfo(
+                                cctx.localNodeId(),
+                                "GridNearOptimisticTxPrepareFuture waiting for remote node response [" +
+                                    "nodeId=" + nodeId +
+                                    ", topVer=" + tx.topologyVersion() +
+                                    ", dhtVer=" + dhtVer +
+                                    ", nearVer=" + nearVer +
+                                    ", futId=" + futId +
+                                    ", miniId=" + miniFut.futId +
+                                    ", tx=" + tx + ']');
+                        }
+                    }
+                    else if (fut instanceof KeyLockFuture) {
+                        KeyLockFuture keyFut = (KeyLockFuture)fut;
+
+                        ctx.basicInfo(
+                            cctx.localNodeId(),
+                            "GridNearOptimisticTxPrepareFuture waiting for local keys lock [" +
+                                "node=" + cctx.localNodeId() +
+                                ", topVer=" + tx.topologyVersion() +
+                                ", allKeysAdded=" + keyFut.allKeysAdded +
+                                ", keys=" + keyFut.lockKeys + ']');
+                    }
+                }
+            }
+        }
+    }
+
+    /** {@inheritDoc} */
     @Override public String toString() {
         Collection<String> futs = F.viewReadOnly(futures(), new C1<IgniteInternalFuture<?>, String>() {
             @Override public String apply(IgniteInternalFuture<?> f) {

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java
index f09b6c8..780674e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java
@@ -177,7 +177,7 @@ public abstract class GridNearOptimisticTxPrepareFutureAdapter extends GridNearT
         protected Collection<IgniteTxKey> lockKeys = new GridConcurrentHashSet<>();
 
         /** */
-        private volatile boolean allKeysAdded;
+        protected volatile boolean allKeysAdded;
 
         /**
          * @param key Key to track for locking.

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareFutureAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareFutureAdapter.java
index ea002f6..1be84c5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareFutureAdapter.java
@@ -27,8 +27,8 @@ import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.processors.cache.GridCacheCompoundFuture;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
-import org.apache.ignite.internal.processors.cache.GridCacheVersionedFuture;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.GridCacheVersionedFuture;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxMapping;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxMapping;
 import org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtDetachedCacheEntry;

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRow.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRow.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRow.java
index 57aeaef..44f0a3f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRow.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRow.java
@@ -24,7 +24,7 @@ import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 /**
  * Cache data row.
  */
-public interface CacheDataRow extends CacheSearchRow {
+public interface CacheDataRow extends CacheSearchRow, Storable {
     /**
      * @return Cache value.
      */

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
index ef148ba..06fe9fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
@@ -72,7 +72,7 @@ public class DataRegionMetricsImpl implements DataRegionMetrics {
 
     /**
      * @param memPlcCfg DataRegionConfiguration.
-    */
+     */
     public DataRegionMetricsImpl(DataRegionConfiguration memPlcCfg) {
         this(memPlcCfg, null);
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java
index 9161d69..1d4b812 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java
@@ -124,7 +124,8 @@ public abstract class DataStructure implements PageLockListener {
      */
     protected final long acquirePage(long pageId) throws IgniteCheckedException {
         assert PageIdUtils.flag(pageId) == FLAG_IDX && PageIdUtils.partId(pageId) == INDEX_PARTITION ||
-            PageIdUtils.flag(pageId) == FLAG_DATA && PageIdUtils.partId(pageId) <= MAX_PARTITION_ID : U.hexLong(pageId);
+            PageIdUtils.flag(pageId) == FLAG_DATA && PageIdUtils.partId(pageId) <= MAX_PARTITION_ID :
+            U.hexLong(pageId) + " flag=" + PageIdUtils.flag(pageId) + " part=" + PageIdUtils.partId(pageId);
 
         return pageMem.acquirePage(grpId, pageId);
     }


[09/17] ignite git commit: IEP-4 Baseline topology for persistent caches (Phase 1) Contributed by: Dmitriy Govorukhin Dmitry Pavlov Eduard Shangareev Ily

Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
index 173ed66..8a04749 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
@@ -18,74 +18,23 @@
 package org.apache.ignite.internal.processors.cache.persistence.tree.io;
 
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.PageUtils;
 import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
-import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.GridStringBuilder;
-import org.apache.ignite.internal.util.typedef.internal.SB;
-import org.jetbrains.annotations.Nullable;
 
 /**
  * Data pages IO.
  */
-public class DataPageIO extends PageIO {
+public class DataPageIO extends AbstractDataPageIO<CacheDataRow> {
     /** */
     public static final IOVersions<DataPageIO> VERSIONS = new IOVersions<>(
         new DataPageIO(1)
     );
 
-    /** */
-    private static final int SHOW_ITEM = 0b0001;
-
-    /** */
-    private static final int SHOW_PAYLOAD_LEN = 0b0010;
-
-    /** */
-    private static final int SHOW_LINK = 0b0100;
-
-    /** */
-    private static final int FREE_LIST_PAGE_ID_OFF = COMMON_HEADER_END;
-
-    /** */
-    private static final int FREE_SPACE_OFF = FREE_LIST_PAGE_ID_OFF + 8;
-
-    /** */
-    private static final int DIRECT_CNT_OFF = FREE_SPACE_OFF + 2;
-
-    /** */
-    private static final int INDIRECT_CNT_OFF = DIRECT_CNT_OFF + 1;
-
-    /** */
-    private static final int FIRST_ENTRY_OFF = INDIRECT_CNT_OFF + 1;
-
-    /** Offset of items (internal page pointers) within data page */
-    public static final int ITEMS_OFF = FIRST_ENTRY_OFF + 2;
-
-    /** */
-    private static final int ITEM_SIZE = 2;
-
-    /** */
-    private static final int PAYLOAD_LEN_SIZE = 2;
-
-    /** */
-    private static final int LINK_SIZE = 8;
-
-    /** */
-    private static final int FRAGMENTED_FLAG = 0b10000000_00000000;
-
-    /** */
-    public static final int MIN_DATA_PAGE_OVERHEAD = ITEMS_OFF + ITEM_SIZE + PAYLOAD_LEN_SIZE + LINK_SIZE;
-
     /**
      * @param ver Page format version.
      */
@@ -94,959 +43,8 @@ public class DataPageIO extends PageIO {
     }
 
     /** {@inheritDoc} */
-    @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
-        super.initNewPage(pageAddr, pageId, pageSize);
-
-        setEmptyPage(pageAddr, pageSize);
-        setFreeListPageId(pageAddr, 0L);
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param pageSize Page size.
-     */
-    private void setEmptyPage(long pageAddr, int pageSize) {
-        setDirectCount(pageAddr, 0);
-        setIndirectCount(pageAddr, 0);
-        setFirstEntryOffset(pageAddr, pageSize, pageSize);
-        setRealFreeSpace(pageAddr, pageSize - ITEMS_OFF, pageSize);
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param freeListPageId Free list page ID.
-     */
-    public void setFreeListPageId(long pageAddr, long freeListPageId) {
-        PageUtils.putLong(pageAddr, FREE_LIST_PAGE_ID_OFF, freeListPageId);
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @return Free list page ID.
-     */
-    public long getFreeListPageId(long pageAddr) {
-        return PageUtils.getLong(pageAddr, FREE_LIST_PAGE_ID_OFF);
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param dataOff Data offset.
-     * @param show What elements of data page entry to show in the result.
-     * @return Data page entry size.
-     */
-    private int getPageEntrySize(long pageAddr, int dataOff, int show) {
-        int payloadLen = PageUtils.getShort(pageAddr, dataOff) & 0xFFFF;
-
-        if ((payloadLen & FRAGMENTED_FLAG) != 0)
-            payloadLen &= ~FRAGMENTED_FLAG; // We are fragmented and have a link.
-        else
-            show &= ~SHOW_LINK; // We are not fragmented, never have a link.
-
-        return getPageEntrySize(payloadLen, show);
-    }
-
-    /**
-     * @param payloadLen Length of the payload, may be a full data row or a row fragment length.
-     * @param show What elements of data page entry to show in the result.
-     * @return Data page entry size.
-     */
-    private int getPageEntrySize(int payloadLen, int show) {
-        assert payloadLen > 0 : payloadLen;
-
-        int res = payloadLen;
-
-        if ((show & SHOW_LINK) != 0)
-            res += LINK_SIZE;
-
-        if ((show & SHOW_ITEM) != 0)
-            res += ITEM_SIZE;
-
-        if ((show & SHOW_PAYLOAD_LEN) != 0)
-            res += PAYLOAD_LEN_SIZE;
-
-        return res;
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param dataOff Entry data offset.
-     * @param pageSize Page size.
-     */
-    private void setFirstEntryOffset(long pageAddr, int dataOff, int pageSize) {
-        assert dataOff >= ITEMS_OFF + ITEM_SIZE && dataOff <= pageSize : dataOff;
-
-        PageUtils.putShort(pageAddr, FIRST_ENTRY_OFF, (short)dataOff);
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @return Entry data offset.
-     */
-    private int getFirstEntryOffset(long pageAddr) {
-        return PageUtils.getShort(pageAddr, FIRST_ENTRY_OFF) & 0xFFFF;
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param freeSpace Free space.
-     * @param pageSize Page size.
-     */
-    private void setRealFreeSpace(long pageAddr, int freeSpace, int pageSize) {
-        assert freeSpace == actualFreeSpace(pageAddr, pageSize) : freeSpace + " != " + actualFreeSpace(pageAddr, pageSize);
-
-        PageUtils.putShort(pageAddr, FREE_SPACE_OFF, (short)freeSpace);
-    }
-
-    /**
-     * Free space refers to a "max row size (without any data page specific overhead) which is
-     * guaranteed to fit into this data page".
-     *
-     * @param pageAddr Page address.
-     * @return Free space.
-     */
-    public int getFreeSpace(long pageAddr) {
-        if (getFreeItemSlots(pageAddr) == 0)
-            return 0;
-
-        int freeSpace = getRealFreeSpace(pageAddr);
-
-        // We reserve size here because of getFreeSpace() method semantics (see method javadoc).
-        // It means that we must be able to accommodate a row of size which is equal to getFreeSpace(),
-        // plus we will have data page overhead: header of the page as well as item, payload length and
-        // possibly a link to the next row fragment.
-        freeSpace -= ITEM_SIZE + PAYLOAD_LEN_SIZE + LINK_SIZE;
-
-        return freeSpace < 0 ? 0 : freeSpace;
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @return {@code true} If there is no useful data in this page.
-     */
-    public boolean isEmpty(long pageAddr) {
-        return getDirectCount(pageAddr) == 0;
-    }
-
-    /**
-     * Equivalent for {@link #actualFreeSpace(long, int)} but reads saved value.
-     *
-     * @param pageAddr Page address.
-     * @return Free space.
-     */
-    private int getRealFreeSpace(long pageAddr) {
-        return PageUtils.getShort(pageAddr, FREE_SPACE_OFF);
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param cnt Direct count.
-     */
-    private void setDirectCount(long pageAddr, int cnt) {
-        assert checkCount(cnt): cnt;
-
-        PageUtils.putByte(pageAddr, DIRECT_CNT_OFF, (byte)cnt);
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @return Direct count.
-     */
-    private int getDirectCount(long pageAddr) {
-        return PageUtils.getByte(pageAddr, DIRECT_CNT_OFF) & 0xFF;
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param c Closure.
-     * @param <T> Closure return type.
-     * @return Collection of closure results for all items in page.
-     * @throws IgniteCheckedException In case of error in closure body.
-     */
-    public <T> List<T> forAllItems(long pageAddr, CC<T> c) throws IgniteCheckedException {
-        long pageId = getPageId(pageAddr);
-
-        int cnt = getDirectCount(pageAddr);
-
-        List<T> res = new ArrayList<>(cnt);
-
-        for (int i = 0; i < cnt; i++) {
-            long link = PageIdUtils.link(pageId, i);
-
-            res.add(c.apply(link));
-        }
-
-        return res;
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param cnt Indirect count.
-     */
-    private void setIndirectCount(long pageAddr, int cnt) {
-        assert checkCount(cnt): cnt;
-
-        PageUtils.putByte(pageAddr, INDIRECT_CNT_OFF, (byte)cnt);
-    }
-
-    /**
-     * @param idx Index.
-     * @return {@code true} If the index is valid.
-     */
-    private boolean checkIndex(int idx) {
-        return idx >= 0 && idx < 0xFF;
-    }
-
-    /**
-     * @param cnt Counter value.
-     * @return {@code true} If the counter fits 1 byte.
-     */
-    private boolean checkCount(int cnt) {
-        return cnt >= 0 && cnt <= 0xFF;
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @return Indirect count.
-     */
-    private int getIndirectCount(long pageAddr) {
-        return PageUtils.getByte(pageAddr, INDIRECT_CNT_OFF) & 0xFF;
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @return Number of free entry slots.
-     */
-    private int getFreeItemSlots(long pageAddr) {
-        return 0xFF - getDirectCount(pageAddr);
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
-     * @param directCnt Direct items count.
-     * @param indirectCnt Indirect items count.
-     * @return Found index of indirect item.
-     */
-    private int findIndirectItemIndex(long pageAddr, int itemId, int directCnt, int indirectCnt) {
-        int low = directCnt;
-        int high = directCnt + indirectCnt - 1;
-
-        while (low <= high) {
-            int mid = (low + high) >>> 1;
-
-            int cmp = Integer.compare(itemId(getItem(pageAddr, mid)), itemId);
-
-            if (cmp < 0)
-                low = mid + 1;
-            else if (cmp > 0)
-                high = mid - 1;
-            else
-                return mid; // found
-        }
-
-        throw new IllegalStateException("Item not found: " + itemId);
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param pageSize Page size.
-     * @return String representation.
-     */
-    private String printPageLayout(long pageAddr, int pageSize) {
-        SB b = new SB();
-
-        printPageLayout(pageAddr, pageSize, b);
-
-        return b.toString();
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param pageSize Page size.
-     * @param b B.
-     */
-    private void printPageLayout(long pageAddr, int pageSize, GridStringBuilder b) {
-        int directCnt = getDirectCount(pageAddr);
-        int indirectCnt = getIndirectCount(pageAddr);
-        int free = getRealFreeSpace(pageAddr);
-
-        boolean valid = directCnt >= indirectCnt;
-
-        b.appendHex(PageIO.getPageId(pageAddr)).a(" [");
-
-        int entriesSize = 0;
-
-        for (int i = 0; i < directCnt; i++) {
-            if (i != 0)
-                b.a(", ");
-
-            short item = getItem(pageAddr, i);
-
-            if (item < ITEMS_OFF || item >= pageSize)
-                valid = false;
-
-            entriesSize += getPageEntrySize(pageAddr, item, SHOW_PAYLOAD_LEN | SHOW_LINK);
-
-            b.a(item);
-        }
-
-        b.a("][");
-
-        Collection<Integer> set = new HashSet<>();
-
-        for (int i = directCnt; i < directCnt + indirectCnt; i++) {
-            if (i != directCnt)
-                b.a(", ");
-
-            short item = getItem(pageAddr, i);
-
-            int itemId = itemId(item);
-            int directIdx = directItemIndex(item);
-
-            if (!set.add(directIdx) || !set.add(itemId))
-                valid = false;
-
-            assert indirectItem(itemId, directIdx) == item;
-
-            if (itemId < directCnt || directIdx < 0 || directIdx >= directCnt)
-                valid = false;
-
-            if (i > directCnt && itemId(getItem(pageAddr, i - 1)) >= itemId)
-                valid = false;
-
-
-            b.a(itemId).a('^').a(directIdx);
-        }
-
-        b.a("][free=").a(free);
-
-        int actualFree = pageSize - ITEMS_OFF - (entriesSize + (directCnt + indirectCnt) * ITEM_SIZE);
-
-        if (free != actualFree) {
-            b.a(", actualFree=").a(actualFree);
-
-            valid = false;
-        }
-        else
-            b.a("]");
-
-        assert valid : b.toString();
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
-     * @param pageSize Page size.
-     * @return Data entry offset in bytes.
-     */
-    private int getDataOffset(long pageAddr, int itemId, int pageSize) {
-        assert checkIndex(itemId): itemId;
-
-        int directCnt = getDirectCount(pageAddr);
-
-        assert directCnt > 0: "itemId=" + itemId + ", directCnt=" + directCnt + ", page=" + printPageLayout(pageAddr, pageSize);
-
-        if (itemId >= directCnt) { // Need to do indirect lookup.
-            int indirectCnt = getIndirectCount(pageAddr);
-
-            // Must have indirect items here.
-            assert indirectCnt > 0: "itemId=" + itemId + ", directCnt=" + directCnt + ", indirectCnt=" + indirectCnt +
-                ", page=" + printPageLayout(pageAddr, pageSize);
-
-            int indirectItemIdx = findIndirectItemIndex(pageAddr, itemId, directCnt, indirectCnt);
-
-            assert indirectItemIdx >= directCnt : indirectItemIdx + " " + directCnt;
-            assert indirectItemIdx < directCnt + indirectCnt: indirectItemIdx + " " + directCnt + " " + indirectCnt;
-
-            itemId = directItemIndex(getItem(pageAddr, indirectItemIdx));
-
-            assert itemId >= 0 && itemId < directCnt: itemId + " " + directCnt + " " + indirectCnt; // Direct item.
-        }
-
-        return directItemToOffset(getItem(pageAddr, itemId));
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param dataOff Points to the entry start.
-     * @return Link to the next entry fragment or 0 if no fragments left or if entry is not fragmented.
-     */
-    private long getNextFragmentLink(long pageAddr, int dataOff) {
-        assert isFragmented(pageAddr, dataOff);
-
-        return PageUtils.getLong(pageAddr, dataOff + PAYLOAD_LEN_SIZE);
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param dataOff Data offset.
-     * @return {@code true} If the data row is fragmented across multiple pages.
-     */
-    private boolean isFragmented(long pageAddr, int dataOff) {
-        return (PageUtils.getShort(pageAddr, dataOff) & FRAGMENTED_FLAG) != 0;
-    }
-
-    /**
-     * Sets position to start of actual fragment data and limit to it's end.
-     *
-     * @param pageAddr Page address.
-     * @param itemId Item to position on.
-     * @param pageSize Page size.
-     * @return {@link DataPagePayload} object.
-     */
-    public DataPagePayload readPayload(final long pageAddr, final int itemId, final int pageSize) {
-        int dataOff = getDataOffset(pageAddr, itemId, pageSize);
-
-        boolean fragmented = isFragmented(pageAddr, dataOff);
-        long nextLink = fragmented ? getNextFragmentLink(pageAddr, dataOff) : 0;
-        int payloadSize = getPageEntrySize(pageAddr, dataOff, 0);
-
-        return new DataPagePayload(dataOff + PAYLOAD_LEN_SIZE + (fragmented ? LINK_SIZE : 0),
-            payloadSize,
-            nextLink);
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param idx Item index.
-     * @return Item.
-     */
-    private short getItem(long pageAddr, int idx) {
-        return PageUtils.getShort(pageAddr, itemOffset(idx));
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param idx Item index.
-     * @param item Item.
-     */
-    private void setItem(long pageAddr, int idx, short item) {
-        PageUtils.putShort(pageAddr, itemOffset(idx), item);
-    }
-
-    /**
-     * @param idx Index of the item.
-     * @return Offset in buffer.
-     */
-    private int itemOffset(int idx) {
-        assert checkIndex(idx): idx;
-
-        return ITEMS_OFF + idx * ITEM_SIZE;
-    }
-
-    /**
-     * @param directItem Direct item.
-     * @return Offset of an entry payload inside of the page.
-     */
-    private int directItemToOffset(short directItem) {
-        return directItem & 0xFFFF;
-    }
-
-    /**
-     * @param dataOff Data offset.
-     * @return Direct item.
-     */
-    private short directItemFromOffset(int dataOff) {
-        assert dataOff >= ITEMS_OFF + ITEM_SIZE && dataOff < Short.MAX_VALUE: dataOff;
-
-        return (short)dataOff;
-    }
-
-    /**
-     * @param indirectItem Indirect item.
-     * @return Index of corresponding direct item.
-     */
-    private int directItemIndex(short indirectItem) {
-        return indirectItem & 0xFF;
-    }
-
-    /**
-     * @param indirectItem Indirect item.
-     * @return Fixed item ID (the index used for referencing an entry from the outside).
-     */
-    private int itemId(short indirectItem) {
-        return (indirectItem & 0xFFFF) >>> 8;
-    }
-
-    /**
-     * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
-     * @param directItemIdx Index of corresponding direct item.
-     * @return Indirect item.
-     */
-    private short indirectItem(int itemId, int directItemIdx) {
-        assert checkIndex(itemId): itemId;
-        assert checkIndex(directItemIdx): directItemIdx;
-
-        return (short)((itemId << 8) | directItemIdx);
-    }
-
-    /**
-     * Move the last direct item to the free slot and reference it with indirect item on the same place.
-     *
-     * @param pageAddr Page address.
-     * @param freeDirectIdx Free slot.
-     * @param directCnt Direct items count.
-     * @param indirectCnt Indirect items count.
-     * @return {@code true} If the last direct item already had corresponding indirect item.
-     */
-    private boolean moveLastItem(long pageAddr, int freeDirectIdx, int directCnt, int indirectCnt) {
-        int lastIndirectId = findIndirectIndexForLastDirect(pageAddr, directCnt, indirectCnt);
-
-        int lastItemId = directCnt - 1;
-
-        assert lastItemId != freeDirectIdx;
-
-        short indirectItem = indirectItem(lastItemId, freeDirectIdx);
-
-        assert itemId(indirectItem) == lastItemId && directItemIndex(indirectItem) == freeDirectIdx;
-
-        setItem(pageAddr, freeDirectIdx, getItem(pageAddr, lastItemId));
-        setItem(pageAddr, lastItemId, indirectItem);
-
-        assert getItem(pageAddr, lastItemId) == indirectItem;
-
-        if (lastIndirectId != -1) { // Fix pointer to direct item.
-            setItem(pageAddr, lastIndirectId, indirectItem(itemId(getItem(pageAddr, lastIndirectId)), freeDirectIdx));
-
-            return true;
-        }
-
-        return false;
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param directCnt Direct items count.
-     * @param indirectCnt Indirect items count.
-     * @return Index of indirect item for the last direct item.
-     */
-    private int findIndirectIndexForLastDirect(long pageAddr, int directCnt, int indirectCnt) {
-        int lastDirectId = directCnt - 1;
-
-        for (int i = directCnt, end = directCnt + indirectCnt; i < end; i++) {
-            short item = getItem(pageAddr, i);
-
-            if (directItemIndex(item) == lastDirectId)
-                return i;
-        }
-
-        return -1;
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param itemId Item ID.
-     * @param pageSize Page size.
-     * @param payload Row data.
-     * @param row Row.
-     * @param rowSize Row size.
-     * @return {@code True} if entry is not fragmented.
-     * @throws IgniteCheckedException If failed.
-     */
-    public boolean updateRow(
-        final long pageAddr,
-        int itemId,
-        int pageSize,
-        @Nullable byte[] payload,
-        @Nullable CacheDataRow row,
-        final int rowSize) throws IgniteCheckedException {
-        assert checkIndex(itemId) : itemId;
-        assert row != null ^ payload != null;
-
-        final int dataOff = getDataOffset(pageAddr, itemId, pageSize);
-
-        if (isFragmented(pageAddr, dataOff))
-            return false;
-
-        if (row != null)
-            writeRowData(pageAddr, dataOff, rowSize, row, false);
-        else
-            writeRowData(pageAddr, dataOff, payload);
-
-        return true;
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
-     * @param pageSize Page size.
-     * @return Next link for fragmented entries or {@code 0} if none.
-     * @throws IgniteCheckedException If failed.
-     */
-    public long removeRow(long pageAddr, int itemId, int pageSize) throws IgniteCheckedException {
-        assert checkIndex(itemId) : itemId;
-
-        final int dataOff = getDataOffset(pageAddr, itemId, pageSize);
-        final long nextLink = isFragmented(pageAddr, dataOff) ? getNextFragmentLink(pageAddr, dataOff) : 0;
-
-        // Record original counts to calculate delta in free space in the end of remove.
-        final int directCnt = getDirectCount(pageAddr);
-        final int indirectCnt = getIndirectCount(pageAddr);
-
-        int curIndirectCnt = indirectCnt;
-
-        assert directCnt > 0 : directCnt; // Direct count always represents overall number of live items.
-
-        // Remove the last item on the page.
-        if (directCnt == 1) {
-            assert (indirectCnt == 0 && itemId == 0) ||
-                (indirectCnt == 1 && itemId == itemId(getItem(pageAddr, 1))) : itemId;
-
-            setEmptyPage(pageAddr, pageSize);
-        }
-        else {
-            // Get the entry size before the actual remove.
-            int rmvEntrySize = getPageEntrySize(pageAddr, dataOff, SHOW_PAYLOAD_LEN | SHOW_LINK);
-
-            int indirectId = 0;
-
-            if (itemId >= directCnt) { // Need to remove indirect item.
-                assert indirectCnt > 0;
-
-                indirectId = findIndirectItemIndex(pageAddr, itemId, directCnt, indirectCnt);
-
-                assert indirectId >= directCnt;
-
-                itemId = directItemIndex(getItem(pageAddr, indirectId));
-
-                assert itemId < directCnt;
-            }
-
-            boolean dropLast = true;
-
-            if (itemId + 1 < directCnt) // It is not the last direct item.
-                dropLast = moveLastItem(pageAddr, itemId, directCnt, indirectCnt);
-
-            if (indirectId == 0) {// For the last direct item with no indirect item.
-                if (dropLast)
-                    moveItems(pageAddr, directCnt, indirectCnt, -1, pageSize);
-                else
-                    curIndirectCnt++;
-            }
-            else {
-                if (dropLast)
-                    moveItems(pageAddr, directCnt, indirectId - directCnt, -1, pageSize);
-
-                moveItems(pageAddr, indirectId + 1, directCnt + indirectCnt - indirectId - 1, dropLast ? -2 : -1, pageSize);
-
-                if (dropLast)
-                    curIndirectCnt--;
-            }
-
-            setIndirectCount(pageAddr, curIndirectCnt);
-            setDirectCount(pageAddr, directCnt - 1);
-
-            assert getIndirectCount(pageAddr) <= getDirectCount(pageAddr);
-
-            // Increase free space.
-            setRealFreeSpace(pageAddr,
-                getRealFreeSpace(pageAddr) + rmvEntrySize + ITEM_SIZE * (directCnt - getDirectCount(pageAddr) + indirectCnt - getIndirectCount(pageAddr)),
-                pageSize);
-        }
-
-        return nextLink;
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param idx Index.
-     * @param cnt Count.
-     * @param step Step.
-     * @param pageSize Page size.
-     */
-    private void moveItems(long pageAddr, int idx, int cnt, int step, int pageSize) {
-        assert cnt >= 0: cnt;
-
-        if (cnt != 0)
-            moveBytes(pageAddr, itemOffset(idx), cnt * ITEM_SIZE, step * ITEM_SIZE, pageSize);
-    }
-
-    /**
-     * @param newEntryFullSize New entry full size (with item, length and link).
-     * @param firstEntryOff First entry data offset.
-     * @param directCnt Direct items count.
-     * @param indirectCnt Indirect items count.
-     * @return {@code true} If there is enough space for the entry.
-     */
-    private boolean isEnoughSpace(int newEntryFullSize, int firstEntryOff, int directCnt, int indirectCnt) {
-        return ITEMS_OFF + ITEM_SIZE * (directCnt + indirectCnt) <= firstEntryOff - newEntryFullSize;
-    }
-
-    /**
-     * Adds row to this data page and sets respective link to the given row object.
-     *
-     * @param pageId page ID.
-     * @param pageAddr Page address.
-     * @param row Cache data row.
-     * @param rowSize Row size.
-     * @param pageSize Page size.
-     * @throws IgniteCheckedException If failed.
-     */
-    public void addRow(
-        final long pageId,
-        final long pageAddr,
-        CacheDataRow row,
-        final int rowSize,
-        final int pageSize
-    ) throws IgniteCheckedException {
-        assert rowSize <= getFreeSpace(pageAddr): "can't call addRow if not enough space for the whole row";
-
-        int fullEntrySize = getPageEntrySize(rowSize, SHOW_PAYLOAD_LEN | SHOW_ITEM);
-
-        int directCnt = getDirectCount(pageAddr);
-        int indirectCnt = getIndirectCount(pageAddr);
-
-        int dataOff = getDataOffsetForWrite(pageAddr, fullEntrySize, directCnt, indirectCnt, pageSize);
-
-        writeRowData(pageAddr, dataOff, rowSize, row, true);
-
-        int itemId = addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
-
-        setLinkByPageId(row, pageId, itemId);
-    }
-
-    /**
-     * Adds row to this data page and sets respective link to the given row object.
-     *
-     * @param pageAddr Page address.
-     * @param payload Payload.
-     * @param pageSize Page size.
-     * @throws IgniteCheckedException If failed.
-     */
-    public void addRow(
-        long pageAddr,
-        byte[] payload,
-        int pageSize
-    ) throws IgniteCheckedException {
-        assert payload.length <= getFreeSpace(pageAddr): "can't call addRow if not enough space for the whole row";
-
-        int fullEntrySize = getPageEntrySize(payload.length, SHOW_PAYLOAD_LEN | SHOW_ITEM);
-
-        int directCnt = getDirectCount(pageAddr);
-        int indirectCnt = getIndirectCount(pageAddr);
-
-        int dataOff = getDataOffsetForWrite(pageAddr, fullEntrySize, directCnt, indirectCnt, pageSize);
-
-        writeRowData(pageAddr, dataOff, payload);
-
-        addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param entryFullSize New entry full size (with item, length and link).
-     * @param directCnt Direct items count.
-     * @param indirectCnt Indirect items count.
-     * @param dataOff First entry offset.
-     * @param pageSize Page size.
-     * @return First entry offset after compaction.
-     */
-    private int compactIfNeed(
-        final long pageAddr,
-        final int entryFullSize,
-        final int directCnt,
-        final int indirectCnt,
-        int dataOff,
-        int pageSize
-    ) {
-        if (!isEnoughSpace(entryFullSize, dataOff, directCnt, indirectCnt)) {
-            dataOff = compactDataEntries(pageAddr, directCnt, pageSize);
-
-            assert isEnoughSpace(entryFullSize, dataOff, directCnt, indirectCnt);
-        }
-
-        return dataOff;
-    }
-
-    /**
-     * Put item reference on entry.
-     *
-     * @param pageAddr Page address.
-     * @param fullEntrySize Full entry size (with link, payload size and item).
-     * @param directCnt Direct items count.
-     * @param indirectCnt Indirect items count.
-     * @param dataOff Data offset.
-     * @param pageSize Page size.
-     * @return Item ID.
-     */
-    private int addItem(final long pageAddr,
-        final int fullEntrySize,
-        final int directCnt,
-        final int indirectCnt,
-        final int dataOff,
-        final int pageSize)
-    {
-        setFirstEntryOffset(pageAddr, dataOff, pageSize);
-
-        int itemId = insertItem(pageAddr, dataOff, directCnt, indirectCnt, pageSize);
-
-        assert checkIndex(itemId): itemId;
-        assert getIndirectCount(pageAddr) <= getDirectCount(pageAddr);
-
-        // Update free space. If number of indirect items changed, then we were able to reuse an item slot.
-        setRealFreeSpace(pageAddr,
-            getRealFreeSpace(pageAddr) - fullEntrySize + (getIndirectCount(pageAddr) != indirectCnt ? ITEM_SIZE : 0),
-            pageSize);
-
-        return itemId;
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param fullEntrySize Full entry size.
-     * @param directCnt Direct items count.
-     * @param indirectCnt Indirect items count.
-     * @param pageSize Page size.
-     * @return Offset in the buffer where the entry must be written.
-     */
-    private int getDataOffsetForWrite(long pageAddr, int fullEntrySize, int directCnt, int indirectCnt, int pageSize) {
-        int dataOff = getFirstEntryOffset(pageAddr);
-
-        // Compact if we do not have enough space for entry.
-        dataOff = compactIfNeed(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
-
-        // We will write data right before the first entry.
-        dataOff -= fullEntrySize - ITEM_SIZE;
-
-        return dataOff;
-    }
-
-    /**
-     * Adds maximum possible fragment of the given row to this data page and sets respective link to the row.
-     *
-     * @param pageMem Page memory.
-     * @param pageAddr Page address.
-     * @param row Cache data row.
-     * @param written Number of bytes of row size that was already written.
-     * @param rowSize Row size.
-     * @param pageSize Page size.
-     * @return Written payload size.
-     * @throws IgniteCheckedException If failed.
-     */
-    public int addRowFragment(
-        PageMemory pageMem,
-        long pageAddr,
-        CacheDataRow row,
-        int written,
-        int rowSize,
-        int pageSize
-    ) throws IgniteCheckedException {
-        return addRowFragment(pageMem, pageAddr, written, rowSize, row.link(), row, null, pageSize);
-    }
-
-    /**
-     * Adds this payload as a fragment to this data page.
-     *
-     * @param pageAddr Page address.
-     * @param payload Payload bytes.
-     * @param lastLink Link to the previous written fragment (link to the tail).
-     * @param pageSize Page size.
-     * @throws IgniteCheckedException If failed.
-     */
-    public void addRowFragment(
-        long pageAddr,
-        byte[] payload,
-        long lastLink,
-        int pageSize
-    ) throws IgniteCheckedException {
-        addRowFragment(null, pageAddr, 0, 0, lastLink, null, payload, pageSize);
-    }
-
-    /**
-     * Adds maximum possible fragment of the given row to this data page and sets respective link to the row.
-     *
-     * @param pageMem Page memory.
-     * @param pageAddr Page address.
-     * @param written Number of bytes of row size that was already written.
-     * @param rowSize Row size.
-     * @param lastLink Link to the previous written fragment (link to the tail).
-     * @param row Row.
-     * @param payload Payload bytes.
-     * @param pageSize Page size.
-     * @return Written payload size.
-     * @throws IgniteCheckedException If failed.
-     */
-    private int addRowFragment(
-        PageMemory pageMem,
-        long pageAddr,
-        int written,
-        int rowSize,
-        long lastLink,
-        CacheDataRow row,
-        byte[] payload,
-        int pageSize
-    ) throws IgniteCheckedException {
-        assert payload == null ^ row == null;
-
-        int directCnt = getDirectCount(pageAddr);
-        int indirectCnt = getIndirectCount(pageAddr);
-
-        int payloadSize = payload != null ? payload.length :
-            Math.min(rowSize - written, getFreeSpace(pageAddr));
-
-        int fullEntrySize = getPageEntrySize(payloadSize, SHOW_PAYLOAD_LEN | SHOW_LINK | SHOW_ITEM);
-        int dataOff = getDataOffsetForWrite(pageAddr, fullEntrySize, directCnt, indirectCnt, pageSize);
-
-        if (payload == null) {
-            ByteBuffer buf = pageMem.pageBuffer(pageAddr);
-
-            buf.position(dataOff);
-
-            short p = (short)(payloadSize | FRAGMENTED_FLAG);
-
-            buf.putShort(p);
-            buf.putLong(lastLink);
-
-            int rowOff = rowSize - written - payloadSize;
-
-            writeFragmentData(row, buf, rowOff, payloadSize);
-        }
-        else {
-            PageUtils.putShort(pageAddr, dataOff, (short)(payloadSize | FRAGMENTED_FLAG));
-
-            PageUtils.putLong(pageAddr, dataOff + 2, lastLink);
-
-            PageUtils.putBytes(pageAddr, dataOff + 10, payload);
-        }
-
-        int itemId = addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
-
-        if (row != null)
-            setLink(row, pageAddr, itemId);
-
-        return payloadSize;
-    }
-
-    /**
-     * @param row Row to set link to.
-     * @param pageAddr Page address.
-     * @param itemId Item ID.
-     */
-    private void setLink(CacheDataRow row, long pageAddr, int itemId) {
-        long pageId = getPageId(pageAddr);
-
-        setLinkByPageId(row, pageId, itemId);
-    }
-
-    /**
-     * @param row Row to set link to.
-     * @param pageId Page ID.
-     * @param itemId Item ID.
-     */
-    private void setLinkByPageId(CacheDataRow row, long pageId, int itemId) {
-        row.link(PageIdUtils.link(pageId, itemId));
-    }
-
-    /**
-     * Write row data fragment.
-     *
-     * @param row Row.
-     * @param buf Byte buffer.
-     * @param rowOff Offset in row data bytes.
-     * @param payloadSize Data length that should be written in a fragment.
-     * @throws IgniteCheckedException If failed.
-     */
-    private void writeFragmentData(
+    @Override
+    protected void writeFragmentData(
         final CacheDataRow row,
         final ByteBuffer buf,
         final int rowOff,
@@ -1243,169 +241,9 @@ public class DataPageIO extends PageIO {
         CACHE_ID
     }
 
-    /**
-     * @param pageAddr Page address.
-     * @param dataOff Data offset.
-     * @param directCnt Direct items count.
-     * @param indirectCnt Indirect items count.
-     * @param pageSize Page size.
-     * @return Item ID (insertion index).
-     */
-    private int insertItem(long pageAddr, int dataOff, int directCnt, int indirectCnt, int pageSize) {
-        if (indirectCnt > 0) {
-            // If the first indirect item is on correct place to become the last direct item, do the transition
-            // and insert the new item into the free slot which was referenced by this first indirect item.
-            short item = getItem(pageAddr, directCnt);
-
-            if (itemId(item) == directCnt) {
-                int directItemIdx = directItemIndex(item);
-
-                setItem(pageAddr, directCnt, getItem(pageAddr, directItemIdx));
-                setItem(pageAddr, directItemIdx, directItemFromOffset(dataOff));
-
-                setDirectCount(pageAddr, directCnt + 1);
-                setIndirectCount(pageAddr, indirectCnt - 1);
-
-                return directItemIdx;
-            }
-        }
-
-        // Move all the indirect items forward to make a free slot and insert new item at the end of direct items.
-        moveItems(pageAddr, directCnt, indirectCnt, +1, pageSize);
-
-        setItem(pageAddr, directCnt, directItemFromOffset(dataOff));
-
-        setDirectCount(pageAddr, directCnt + 1);
-        assert getDirectCount(pageAddr) == directCnt + 1;
-
-        return directCnt; // Previous directCnt will be our itemId.
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param directCnt Direct items count.
-     * @param pageSize Page size.
-     * @return New first entry offset.
-     */
-    private int compactDataEntries(long pageAddr, int directCnt, int pageSize) {
-        assert checkCount(directCnt): directCnt;
-
-        int[] offs = new int[directCnt];
-
-        for (int i = 0; i < directCnt; i++) {
-            int off = directItemToOffset(getItem(pageAddr, i));
-
-            offs[i] = (off << 8) | i; // This way we'll be able to sort by offset using Arrays.sort(...).
-        }
-
-        Arrays.sort(offs);
-
-        // Move right all of the entries if possible to make the page as compact as possible to its tail.
-        int prevOff = pageSize;
-
-        final int start = directCnt - 1;
-        int curOff = offs[start] >>> 8;
-        int curEntrySize = getPageEntrySize(pageAddr, curOff, SHOW_PAYLOAD_LEN | SHOW_LINK);
-
-        for (int i = start; i >= 0; i--) {
-            assert curOff < prevOff : curOff;
-
-            int delta = prevOff - (curOff + curEntrySize);
-
-            int off = curOff;
-            int entrySize = curEntrySize;
-
-            if (delta != 0) { // Move right.
-                assert delta > 0: delta;
-
-                int itemId = offs[i] & 0xFF;
-
-                setItem(pageAddr, itemId, directItemFromOffset(curOff + delta));
-
-                for (int j = i - 1; j >= 0; j--) {
-                    int offNext = offs[j] >>> 8;
-                    int nextSize = getPageEntrySize(pageAddr, offNext, SHOW_PAYLOAD_LEN | SHOW_LINK);
-
-                    if (offNext + nextSize == off) {
-                        i--;
-
-                        off = offNext;
-                        entrySize += nextSize;
-
-                        itemId = offs[j] & 0xFF;
-                        setItem(pageAddr, itemId, directItemFromOffset(offNext + delta));
-                    }
-                    else {
-                        curOff = offNext;
-                        curEntrySize = nextSize;
-
-                        break;
-                    }
-                }
-
-                moveBytes(pageAddr, off, entrySize, delta, pageSize);
-
-                off += delta;
-            }
-            else if (i > 0) {
-                curOff = offs[i - 1] >>> 8;
-                curEntrySize = getPageEntrySize(pageAddr, curOff, SHOW_PAYLOAD_LEN | SHOW_LINK);
-            }
-
-            prevOff = off;
-        }
-
-        return prevOff;
-    }
-
-    /**
-     * Full-scan free space calculation procedure.
-     *
-     * @param pageAddr Page to scan.
-     * @param pageSize Page size.
-     * @return Actual free space in the buffer.
-     */
-    private int actualFreeSpace(long pageAddr, int pageSize) {
-        int directCnt = getDirectCount(pageAddr);
-
-        int entriesSize = 0;
-
-        for (int i = 0; i < directCnt; i++) {
-            int off = directItemToOffset(getItem(pageAddr, i));
-
-            int entrySize = getPageEntrySize(pageAddr, off, SHOW_PAYLOAD_LEN | SHOW_LINK);
-
-            entriesSize += entrySize;
-        }
-
-        return pageSize - ITEMS_OFF - entriesSize - (directCnt + getIndirectCount(pageAddr)) * ITEM_SIZE;
-    }
-
-    /**
-     * @param addr Address.
-     * @param off Offset.
-     * @param cnt Count.
-     * @param step Step.
-     * @param pageSize Page size.
-     */
-    private void moveBytes(long addr, int off, int cnt, int step, int pageSize) {
-        assert step != 0: step;
-        assert off + step >= 0;
-        assert off + step + cnt <= pageSize : "[off=" + off + ", step=" + step + ", cnt=" + cnt +
-            ", cap=" + pageSize + ']';
-
-        PageHandler.copyMemory(addr, off, addr, off + step, cnt);
-    }
-
-    /**
-     * @param pageAddr Page address.
-     * @param dataOff Data offset.
-     * @param payloadSize Payload size.
-     * @param row Data row.
-     * @param newRow {@code False} if existing cache entry is updated, in this case skip key data write.
-     * @throws IgniteCheckedException If failed.
-     */
-    private void writeRowData(
+    /** {@inheritDoc} */
+    @Override
+    protected void writeRowData(
         long pageAddr,
         int dataOff,
         int payloadSize,
@@ -1439,12 +277,9 @@ public class DataPageIO extends PageIO {
         PageUtils.putLong(addr, 0, row.expireTime());
     }
 
-    /**
-     * @param pageAddr Page address.
-     * @param dataOff Data offset.
-     * @param payload Payload
-     */
-    private void writeRowData(
+    /** {@inheritDoc} */
+    @Override
+    protected void writeRowData(
         long pageAddr,
         int dataOff,
         byte[] payload
@@ -1456,6 +291,11 @@ public class DataPageIO extends PageIO {
     }
 
     /** {@inheritDoc} */
+    @Override public int getRowSize(CacheDataRow row) throws IgniteCheckedException {
+        return getRowSize(row, row.cacheId() != 0);
+    }
+
+    /** {@inheritDoc} */
     @Override protected void printPage(long addr, int pageSize, GridStringBuilder sb) throws IgniteCheckedException {
         sb.a("DataPageIO [\n");
         printPageLayout(addr, pageSize, sb);
@@ -1463,18 +303,18 @@ public class DataPageIO extends PageIO {
     }
 
     /**
-     * Defines closure interface for applying computations to data page items.
-     *
-     * @param <T> Closure return type.
+     * @param row Row.
+     * @param withCacheId If {@code true} adds cache ID size.
+     * @return Entry size on page.
+     * @throws IgniteCheckedException If failed.
      */
-    public interface CC<T> {
-        /**
-         * Closure body.
-         *
-         * @param link Link to item.
-         * @return Closure return value.
-         * @throws IgniteCheckedException In case of error in closure body.
-         */
-        public T apply(long link) throws IgniteCheckedException;
+    public static int getRowSize(CacheDataRow row, boolean withCacheId) throws IgniteCheckedException {
+        KeyCacheObject key = row.key();
+        CacheObject val = row.value();
+
+        int keyLen = key.valueBytesLength(null);
+        int valLen = val.valueBytesLength(null);
+
+        return keyLen + valLen + CacheVersionIO.size(row.version(), false) + 8 + (withCacheId ? 4 : 0);
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java
index 60b1aaf..a5236c2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java
@@ -25,9 +25,10 @@ import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.PageUtils;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.processors.cache.persistence.MetadataStorage;
+import org.apache.ignite.internal.processors.cache.persistence.IndexStorageImpl;
 import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListMetaIO;
 import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListNodeIO;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageTree;
 import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
 import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageLockListener;
 import org.apache.ignite.internal.processors.cache.tree.CacheIdAwareDataInnerIO;
@@ -181,6 +182,16 @@ public abstract class PageIO {
     /** */
     public static final short T_PART_CNTRS = 20;
 
+    /** */
+    public static final short T_DATA_METASTORAGE = 21;
+
+    /** */
+    public static final short T_DATA_REF_METASTORAGE_INNER = 22;
+
+    /** */
+    public static final short T_DATA_REF_METASTORAGE_LEAF = 23;
+
+
     /** Index for payload == 1. */
     public static final short T_H2_EX_REF_LEAF_START = 10000;
 
@@ -460,6 +471,9 @@ public abstract class PageIO {
             case T_PAGE_UPDATE_TRACKING:
                 return (Q)TrackingPageIO.VERSIONS.forVersion(ver);
 
+            case T_DATA_METASTORAGE:
+                return (Q)SimpleDataPageIO.VERSIONS.forVersion(ver);
+
             default:
                 return (Q)getBPlusIO(type, ver);
         }
@@ -518,10 +532,10 @@ public abstract class PageIO {
                 return (Q)CacheIdAwareDataLeafIO.VERSIONS.forVersion(ver);
 
             case T_METASTORE_INNER:
-                return (Q)MetadataStorage.MetaStoreInnerIO.VERSIONS.forVersion(ver);
+                return (Q)IndexStorageImpl.MetaStoreInnerIO.VERSIONS.forVersion(ver);
 
             case T_METASTORE_LEAF:
-                return (Q)MetadataStorage.MetaStoreLeafIO.VERSIONS.forVersion(ver);
+                return (Q)IndexStorageImpl.MetaStoreLeafIO.VERSIONS.forVersion(ver);
 
             case T_PENDING_REF_INNER:
                 return (Q)PendingEntryInnerIO.VERSIONS.forVersion(ver);
@@ -535,6 +549,12 @@ public abstract class PageIO {
             case T_CACHE_ID_AWARE_PENDING_REF_LEAF:
                 return (Q)CacheIdAwarePendingEntryLeafIO.VERSIONS.forVersion(ver);
 
+            case T_DATA_REF_METASTORAGE_INNER:
+                return (Q)MetastorageTree.MetastorageInnerIO.VERSIONS.forVersion(ver);
+
+            case T_DATA_REF_METASTORAGE_LEAF:
+                return (Q)MetastorageTree.MetastoreLeafIO.VERSIONS.forVersion(ver);
+
             default:
                 // For tests.
                 if (innerTestIO != null && innerTestIO.getType() == type && innerTestIO.getVersion() == ver)

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionCountersIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionCountersIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionCountersIO.java
index 4803b44..68e6e2f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionCountersIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionCountersIO.java
@@ -23,9 +23,11 @@ import java.util.Map;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.pagemem.PageUtils;
 import org.apache.ignite.internal.util.GridStringBuilder;
+import org.apache.ignite.internal.util.GridUnsafe;
 
 /**
- *
+ * Page IO for Partition Counters, IO for pages containing cache ID mapping to its size. Used only for caches in shared
+ * cache groups.
  */
 public class PagePartitionCountersIO extends PageIO {
     /** */
@@ -40,8 +42,14 @@ public class PagePartitionCountersIO extends PageIO {
     /** */
     private static final int ITEMS_OFF = NEXT_COUNTERS_PAGE_OFF + 8;
 
-    /** */
-    private static final int ITEM_SIZE = 12;
+    /** Serialized size in bytes of cache ID (int) */
+    private static final int CACHE_ID_SIZE = 4;
+
+    /** Serialized size in bytes of cache ID (int) */
+    private static final int CACHE_SIZE_SIZE = 8;
+
+    /** One serialized entry size: Item size = 4 bytes (cache ID) + 8 bytes (cache size) = 12 bytes */
+    public static final int ITEM_SIZE = CACHE_ID_SIZE + CACHE_SIZE_SIZE;
 
     /** */
     private static final byte LAST_FLAG = 0b1;
@@ -58,6 +66,22 @@ public class PagePartitionCountersIO extends PageIO {
         super(T_PART_CNTRS, ver);
     }
 
+    /**
+     * @param cacheSizes Cache sizes: cache Id in shared group mapped to its size. Not null.
+     * @return Serialized cache sizes or 0-byte length array if map was empty.
+     */
+    public byte[] serializeCacheSizes(Map<Integer, Long> cacheSizes) {
+        byte[] data = new byte[cacheSizes.size() * ITEM_SIZE];
+        long off = GridUnsafe.BYTE_ARR_OFF;
+
+        for (Map.Entry<Integer, Long> entry : cacheSizes.entrySet()) {
+            GridUnsafe.putInt(data, off, entry.getKey()); off += CACHE_ID_SIZE;
+            GridUnsafe.putLong(data, off, entry.getValue()); off += CACHE_SIZE_SIZE;
+        }
+
+        return data;
+    }
+
     /** {@inheritDoc} */
     @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
         super.initNewPage(pageAddr, pageId, pageSize);
@@ -125,12 +149,12 @@ public class PagePartitionCountersIO extends PageIO {
 
         for (int i = 0; i < cnt; i++) {
             int cacheId = PageUtils.getInt(pageAddr, off);
-            off += 4;
+            off += CACHE_ID_SIZE;
 
             assert cacheId != 0;
 
             long cacheSize = PageUtils.getLong(pageAddr, off);
-            off += 8;
+            off += CACHE_SIZE_SIZE;
 
             assert cacheSize >= 0 : cacheSize;
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java
index 9a50e5c..3d79884 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java
@@ -24,7 +24,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartit
 import org.apache.ignite.internal.util.GridStringBuilder;
 
 /**
- *
+ * IO for partition metadata pages.
  */
 public class PagePartitionMetaIO extends PageMetaIO {
     /** */
@@ -136,7 +136,7 @@ public class PagePartitionMetaIO extends PageMetaIO {
     }
 
     /**
-     * @param pageAddr Page address
+     * @param pageAddr Partition metadata page address.
      * @param state State.
      */
     public boolean setPartitionState(long pageAddr, byte state) {
@@ -149,7 +149,8 @@ public class PagePartitionMetaIO extends PageMetaIO {
     }
 
     /**
-     * @param pageAddr Page address.
+     * Returns partition counters page identifier, page with caches in cache group sizes.
+     * @param pageAddr Partition metadata page address.
      * @return Next meta partial page ID or {@code 0} if it does not exist.
      */
     public long getCountersPageId(long pageAddr) {
@@ -157,11 +158,13 @@ public class PagePartitionMetaIO extends PageMetaIO {
     }
 
     /**
-     * @param pageAddr Page address.
-     * @param metaPageId Next partial meta page ID.
+     * Sets new reference to partition counters page (logical cache sizes).
+     *
+     * @param pageAddr Partition metadata page address.
+     * @param cntrsPageId New cache sizes page ID.
      */
-    public void setCountersPageId(long pageAddr, long metaPageId) {
-        PageUtils.putLong(pageAddr, NEXT_PART_META_PAGE_OFF, metaPageId);
+    public void setCountersPageId(long pageAddr, long cntrsPageId) {
+        PageUtils.putLong(pageAddr, NEXT_PART_META_PAGE_OFF, cntrsPageId);
     }
 
     /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/SimpleDataPageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/SimpleDataPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/SimpleDataPageIO.java
new file mode 100644
index 0000000..f02e3a0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/SimpleDataPageIO.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageDataRow;
+import org.apache.ignite.internal.util.GridStringBuilder;
+
+/**
+ * Data pages IO for Metastorage.
+ */
+public class SimpleDataPageIO extends AbstractDataPageIO<MetastorageDataRow> {
+    /** */
+    public static final IOVersions<SimpleDataPageIO> VERSIONS = new IOVersions<>(
+        new SimpleDataPageIO(1)
+    );
+
+    /**
+     * @param ver Page format version.
+     */
+    public SimpleDataPageIO(int ver) {
+        super(T_DATA_METASTORAGE, ver);
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    protected void writeFragmentData(
+        final MetastorageDataRow row,
+        final ByteBuffer buf,
+        final int rowOff,
+        final int payloadSize
+    ) throws IgniteCheckedException {
+        int written = writeSizeFragment(row, buf, rowOff, payloadSize);
+
+        if (payloadSize == written)
+            return;
+
+        int start = rowOff > 4 ? rowOff - 4 : 0;
+
+        final int len = Math.min(row.value().length - start, payloadSize - written);
+
+        if (len > 0) {
+            buf.put(row.value(), start, len);
+            written += len;
+        }
+
+        assert written == payloadSize;
+    }
+
+    /** */
+    private int writeSizeFragment(final MetastorageDataRow row, final ByteBuffer buf, final int rowOff,
+        final int payloadSize) {
+        final int size = 4;
+
+        if (rowOff >= size)
+            return 0;
+
+        if (rowOff == 0 && payloadSize >= size) {
+            buf.putInt(row.value().length);
+
+            return size;
+        }
+
+        ByteBuffer buf2 = ByteBuffer.allocate(size);
+        buf2.order(buf.order());
+
+        buf2.putInt(row.value().length);
+        int len = Math.min(size - rowOff, payloadSize);
+        buf.put(buf2.array(), rowOff, len);
+
+        return len;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    protected void writeRowData(
+        long pageAddr,
+        int dataOff,
+        int payloadSize,
+        MetastorageDataRow row,
+        boolean newRow
+    ) throws IgniteCheckedException {
+        long addr = pageAddr + dataOff;
+
+        if (newRow)
+            PageUtils.putShort(addr, 0, (short)payloadSize);
+
+        PageUtils.putInt(addr, 2, row.value().length);
+        PageUtils.putBytes(addr, 6, row.value());
+    }
+
+    public static byte[] readPayload(long link) {
+        int size = PageUtils.getInt(link, 0);
+
+        return PageUtils.getBytes(link, 4, size);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int getRowSize(MetastorageDataRow row) throws IgniteCheckedException {
+        return 4 + row.value().length;
+    }
+
+
+    /** {@inheritDoc} */
+    @Override protected void printPage(long addr, int pageSize, GridStringBuilder sb) throws IgniteCheckedException {
+        sb.a("SimpleDataPageIO [\n");
+        printPageLayout(addr, pageSize, sb);
+        sb.a("\n]");
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
index c2c07f6..195d181 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
@@ -30,6 +30,7 @@ import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.UnzipFileIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactory;
 import org.apache.ignite.internal.util.GridCloseableIteratorAdapter;
 import org.apache.ignite.internal.util.typedef.P2;
@@ -219,22 +220,22 @@ public abstract class AbstractWalRecordsIterator
         if (hnd == null)
             return null;
 
-        FileWALPointer ptr = new FileWALPointer(hnd.idx, (int)hnd.in.position(),0);
+        FileWALPointer actualFilePtr = new FileWALPointer(hnd.idx, (int)hnd.in.position(), 0);
 
         try {
-            WALRecord rec = hnd.ser.readRecord(hnd.in, ptr);
+            WALRecord rec = hnd.ser.readRecord(hnd.in, actualFilePtr);
 
-            ptr.length(rec.size());
+            actualFilePtr.length(rec.size());
 
             // cast using diamond operator here can break compile for 7
-            return new IgniteBiTuple<>((WALPointer)ptr, postProcessRecord(rec));
+            return new IgniteBiTuple<>((WALPointer)actualFilePtr, postProcessRecord(rec));
         }
         catch (IOException | IgniteCheckedException e) {
             if (e instanceof WalSegmentTailReachedException)
                 throw (WalSegmentTailReachedException)e;
 
             if (!(e instanceof SegmentEofException))
-                handleRecordException(e, ptr);
+                handleRecordException(e, actualFilePtr);
 
             return null;
         }
@@ -260,7 +261,7 @@ public abstract class AbstractWalRecordsIterator
         @NotNull final Exception e,
         @Nullable final FileWALPointer ptr) {
         if (log.isInfoEnabled())
-            log.info("Stopping WAL iteration due to an exception: " + e.getMessage());
+            log.info("Stopping WAL iteration due to an exception: " + e.getMessage() + ", ptr=" + ptr);
     }
 
     /**
@@ -275,7 +276,7 @@ public abstract class AbstractWalRecordsIterator
         @Nullable final FileWALPointer start)
         throws IgniteCheckedException, FileNotFoundException {
         try {
-            FileIO fileIO = ioFactory.create(desc.file);
+            FileIO fileIO = desc.isCompressed() ? new UnzipFileIO(desc.file) : ioFactory.create(desc.file);
 
             try {
                 IgniteBiTuple<Integer, Boolean> tup = FileWriteAheadLogManager.readSerializerVersionAndCompactedFlag(fileIO);
@@ -284,12 +285,13 @@ public abstract class AbstractWalRecordsIterator
 
                 boolean isCompacted = tup.get2();
 
+                if (isCompacted)
+                    serializerFactory.skipPositionCheck(true);
+
                 FileInput in = new FileInput(fileIO, buf);
 
                 if (start != null && desc.idx == start.index()) {
                     if (isCompacted) {
-                        serializerFactory.skipPositionCheck(true);
-
                         if (start.fileOffset() != 0)
                             serializerFactory.recordDeserializeFilter(new StartSeekingFilter(start));
                     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 8c27af0..d29774c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -35,7 +35,9 @@ import java.nio.MappedByteBuffer;
 import java.nio.channels.ClosedByInterruptException;
 import java.nio.file.Files;
 import java.sql.Time;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -95,11 +97,13 @@ import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.CIX1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.lang.IgniteUuid;
 import org.jetbrains.annotations.NotNull;
@@ -120,6 +124,7 @@ import static org.apache.ignite.internal.util.IgniteUtils.findNonPublicMethod;
 /**
  * File WAL manager.
  */
+@SuppressWarnings("IfMayBeConditional")
 public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter implements IgniteWriteAheadLogManager {
     /** {@link MappedByteBuffer#force0(java.io.FileDescriptor, long, long)}. */
     private static final Method force0 = findNonPublicMethod(
@@ -258,12 +263,15 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     private final int serializerVer =
         IgniteSystemProperties.getInteger(IGNITE_WAL_SERIALIZER_VERSION, LATEST_SERIALIZER_VERSION);
 
-    /** Latest segment cleared by {@link #truncate(WALPointer)}. */
+    /** Latest segment cleared by {@link #truncate(WALPointer, WALPointer)}. */
     private volatile long lastTruncatedArchiveIdx = -1L;
 
     /** Factory to provide I/O interfaces for read/write operations with files */
     private final FileIOFactory ioFactory;
 
+    /** Next segment archived monitor. */
+    private final Object nextSegmentArchivedMonitor = new Object();
+
     /** Updater for {@link #currHnd}, used for verify there are no concurrent update for current log segment handle */
     private static final AtomicReferenceFieldUpdater<FileWriteAheadLogManager, FileWriteHandle> CURR_HND_UPD =
         AtomicReferenceFieldUpdater.newUpdater(FileWriteAheadLogManager.class, FileWriteHandle.class, "currHnd");
@@ -374,6 +382,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
                 compressor = new FileCompressor();
 
                 decompressor = new FileDecompressor();
+
+                if (!cctx.kernalContext().clientNode())
+                    decompressor.start();
             }
 
             if (mode != WALMode.NONE) {
@@ -387,6 +398,54 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     }
 
     /**
+     *
+     */
+    public Collection<File> getAndReserveWalFiles(FileWALPointer low, FileWALPointer high) throws IgniteCheckedException {
+        FileArchiver archiver0 = archiver;
+
+        final long awaitIdx = high.index() - 1;
+
+        awaitSegmentArchived(archiver0, awaitIdx);
+
+        if (!reserve(low))
+            throw new IgniteCheckedException("WAL archive segment has been deleted [idx=" + low.index() + "]");
+
+        List<File> res = new ArrayList<>();
+
+        for (long i = low.index(); i < high.index(); i++) {
+            String segmentName = FileDescriptor.fileName(i);
+
+            File file = new File(walArchiveDir, segmentName);
+
+            if (file.exists())
+                res.add(file);
+            else if ((file = new File(walArchiveDir, segmentName + ".zip")).exists())
+                res.add(file);
+            else
+                throw new IgniteCheckedException("WAL archive segment has been deleted [idx=" + i + "]");
+        }
+
+        return res;
+    }
+
+    /**
+     * @param archiver0 Archiver.
+     * @param awaitIdx Method will wait archivation of that index.
+     */
+    private void awaitSegmentArchived(FileArchiver archiver0, long awaitIdx) throws IgniteInterruptedCheckedException {
+        synchronized (nextSegmentArchivedMonitor) {
+            while (archiver0.lastArchivedAbsoluteIndex() < awaitIdx) {
+                try {
+                    nextSegmentArchivedMonitor.wait(2000);
+                }
+                catch (InterruptedException e) {
+                    throw new IgniteInterruptedCheckedException(e);
+                }
+            }
+        }
+    }
+
+    /**
      * @throws IgniteCheckedException if WAL store path is configured and archive path isn't (or vice versa)
      */
     private void checkWalConfiguration() throws IgniteCheckedException {
@@ -453,9 +512,6 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
 
             if (compressor != null)
                 compressor.start();
-
-            if (decompressor != null)
-                decompressor.start();
         }
     }
 
@@ -616,7 +672,20 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
         // Need to calculate record size first.
         rec.size(serializer.size(rec));
 
-        for (; ; currWrHandle = rollOver(currWrHandle)) {
+        while (true) {
+            if (rec.rollOver()){
+                assert cctx.database().checkpointLockIsHeldByThread();
+
+                long idx = currWrHandle.idx;
+
+                currWrHandle.buf.close();
+
+                currWrHandle = rollOver(currWrHandle);
+
+                if (log != null && log.isDebugEnabled())
+                    log.debug("Rollover segment [" + idx + " to " + currWrHandle.idx + "], recordType=" + rec.type());
+            }
+
             WALPointer ptr = currWrHandle.addRecord(rec);
 
             if (ptr != null) {
@@ -629,6 +698,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
 
                 return ptr;
             }
+            else
+                currWrHandle = rollOver(currWrHandle);
 
             checkEnvironment();
 
@@ -754,14 +825,15 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     }
 
     /** {@inheritDoc} */
-    @Override public int truncate(WALPointer ptr) {
-        if (ptr == null)
+    @Override public int truncate(WALPointer low, WALPointer high) {
+        if (high == null)
             return 0;
 
-        assert ptr instanceof FileWALPointer : ptr;
+        assert high instanceof FileWALPointer : high;
 
         // File pointer bound: older entries will be deleted from archive
-        FileWALPointer fPtr = (FileWALPointer)ptr;
+        FileWALPointer lowPtr = (FileWALPointer)low;
+        FileWALPointer highPtr = (FileWALPointer)high;
 
         FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER));
 
@@ -770,12 +842,17 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
         FileArchiver archiver0 = archiver;
 
         for (FileDescriptor desc : descs) {
+            if (lowPtr != null && desc.idx < lowPtr.index())
+                continue;
+
             // Do not delete reserved or locked segment and any segment after it.
             if (archiver0 != null && archiver0.reserved(desc.idx))
                 return deleted;
 
+            long lastArchived = archiver0 != null ? archiver0.lastArchivedAbsoluteIndex() : lastArchivedIndex();
+
             // We need to leave at least one archived segment to correctly determine the archive index.
-            if (desc.idx + 1 < fPtr.index()) {
+            if (desc.idx < highPtr.index() && desc.idx < lastArchived) {
                 if (!desc.file.delete())
                     U.warn(log, "Failed to remove obsolete WAL segment (make sure the process has enough rights): " +
                         desc.file.getAbsolutePath());
@@ -1085,7 +1162,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
 
                     break;
                 }
-                catch (ClosedByInterruptException e) {
+                catch (ClosedByInterruptException ignore) {
                     interrupted = true;
 
                     Thread.interrupted();
@@ -1151,7 +1228,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
             createFile(first);
         }
         else
-            checkFiles(0, false, null);
+            checkFiles(0, false, null, null);
     }
 
     /**
@@ -1285,9 +1362,15 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
         /** current thread stopping advice */
         private volatile boolean stopped;
 
-        /** */
+        /**
+         * Maps absolute segment index to reservation counter. If counter > 0 then we wouldn't delete all segments
+         * which >= reserved segment index.
+         */
         private NavigableMap<Long, Integer> reserved = new TreeMap<>();
 
+        /** Formatted index. */
+        private int formatted;
+
         /**
          * Maps absolute segment index to locks counter. Lock on segment protects from archiving segment and may come
          * from {@link RecordsIterator} during WAL replay. Map itself is guarded by <code>this</code>.
@@ -1391,8 +1474,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
                     while (curAbsWalIdx == -1 && !stopped)
                         wait();
 
-                    if (curAbsWalIdx != 0 && lastAbsArchivedIdx == -1)
-                        changeLastArchivedIndexAndWakeupCompressor(curAbsWalIdx - 1);
+                    // If the archive directory is empty, we can be sure that there were no WAL segments archived.
+                    // This is ensured by the check in truncate() which will leave at least one file there
+                    // once it was archived.
                 }
 
                 while (!Thread.currentThread().isInterrupted() && !stopped) {
@@ -1425,7 +1509,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
 
                         synchronized (this) {
                             // Then increase counter to allow rollover on clean working file
-                            changeLastArchivedIndexAndWakeupCompressor(toArchive);
+                            changeLastArchivedIndexAndNotifyWaiters(toArchive);
 
                             notifyAll();
                         }
@@ -1451,11 +1535,15 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
         /**
          * @param idx Index.
          */
-        private void changeLastArchivedIndexAndWakeupCompressor(long idx) {
+        private void changeLastArchivedIndexAndNotifyWaiters(long idx) {
             lastAbsArchivedIdx = idx;
 
             if (compressor != null)
                 compressor.onNextSegmentArchived();
+
+            synchronized (nextSegmentArchivedMonitor) {
+                nextSegmentArchivedMonitor.notifyAll();
+            }
         }
 
         /**
@@ -1482,11 +1570,20 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
                     try {
                         wait();
                     }
-                    catch (InterruptedException e) {
+                    catch (InterruptedException ignore) {
                         interrupted.set(true);
                     }
                 }
 
+                // Wait for formatter so that we do not open an empty file in DEFAULT mode.
+                while (curAbsWalIdx % dsCfg.getWalSegments() > formatted)
+                    try {
+                        wait();
+                    }
+                    catch (InterruptedException ignore) {
+                        interrupted.set(true);
+                    }
+
                 return curAbsWalIdx;
             }
         }
@@ -1606,11 +1703,24 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
          * FileWriteAheadLogManager#checkOrPrepareFiles()}
          */
         private void allocateRemainingFiles() throws IgniteCheckedException {
-            checkFiles(1, true, new IgnitePredicate<Integer>() {
-                @Override public boolean apply(Integer integer) {
-                    return !checkStop();
+            checkFiles(
+                1,
+                true,
+                new IgnitePredicate<Integer>() {
+                    @Override public boolean apply(Integer integer) {
+                        return !checkStop();
+                    }
+                },
+                new CI1<Integer>() {
+                    @Override public void apply(Integer idx) {
+                        synchronized (FileArchiver.this) {
+                            formatted = idx;
+
+                            FileArchiver.this.notifyAll();
+                        }
+                    }
                 }
-            });
+            );
         }
     }
 
@@ -1754,7 +1864,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
                     if (handle != null)
                         handle.invalidateEnvironment(e);
                 }
-                catch (InterruptedException e) {
+                catch (InterruptedException ignore) {
                     Thread.currentThread().interrupt();
                 }
             }
@@ -1802,6 +1912,12 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
                     while (iter.hasNextX())
                         iter.nextX();
                 }
+
+                RecordSerializer ser = new RecordSerializerFactoryImpl(cctx).createSerializer(segmentSerializerVer);
+
+                ByteBuffer heapBuf = prepareSwitchSegmentRecordBuffer(nextSegment, ser);
+
+                zos.write(heapBuf.array());
             }
             finally {
                 release(new FileWALPointer(nextSegment, 0, 0));
@@ -1809,6 +1925,25 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
         }
 
         /**
+         * @param nextSegment Segment index.
+         * @param ser Record Serializer.
+         */
+        @NotNull private ByteBuffer prepareSwitchSegmentRecordBuffer(long nextSegment, RecordSerializer ser)
+            throws IgniteCheckedException {
+            SwitchSegmentRecord switchRecord = new SwitchSegmentRecord();
+
+            int switchRecordSize = ser.size(switchRecord);
+            switchRecord.size(switchRecordSize);
+
+            switchRecord.position(new FileWALPointer(nextSegment, 0, switchRecordSize));
+
+            ByteBuffer heapBuf = ByteBuffer.allocate(switchRecordSize);
+
+            ser.writeRecord(switchRecord, heapBuf);
+            return heapBuf;
+        }
+
+        /**
          * @throws IgniteInterruptedCheckedException If failed to wait for thread shutdown.
          */
         private void shutdown() throws IgniteInterruptedCheckedException {
@@ -1833,7 +1968,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
         private Map<Long, GridFutureAdapter<Void>> decompressionFutures = new HashMap<>();
 
         /** Segments queue. */
-        private PriorityBlockingQueue<Long> segmentsQueue = new PriorityBlockingQueue<>();
+        private final PriorityBlockingQueue<Long> segmentsQueue = new PriorityBlockingQueue<>();
 
         /** Byte array for draining data. */
         private byte[] arr = new byte[BUF_SIZE];
@@ -1873,7 +2008,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
                         decompressionFutures.remove(segmentToDecompress).onDone();
                     }
                 }
-                catch (InterruptedException e){
+                catch (InterruptedException ignore) {
                     Thread.currentThread().interrupt();
                 }
                 catch (IOException e) {
@@ -1934,7 +2069,12 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
      * @param p Predicate Exit condition.
      * @throws IgniteCheckedException if validation or create file fail.
      */
-    private void checkFiles(int startWith, boolean create, IgnitePredicate<Integer> p) throws IgniteCheckedException {
+    private void checkFiles(
+        int startWith,
+        boolean create,
+        @Nullable IgnitePredicate<Integer> p,
+        @Nullable IgniteInClosure<Integer> completionCallback
+    ) throws IgniteCheckedException {
         for (int i = startWith; i < dsCfg.getWalSegments() && (p == null || p.apply(i)); i++) {
             File checkFile = new File(walWorkDir, FileDescriptor.fileName(i));
 
@@ -1944,10 +2084,16 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
                         "the same name already exists): " + checkFile.getAbsolutePath());
                 else if (checkFile.length() != dsCfg.getWalSegmentSize() && mode == WALMode.DEFAULT)
                     throw new IgniteCheckedException("Failed to initialize WAL log segment " +
-                        "(WAL segment size change is not supported):" + checkFile.getAbsolutePath());
+                        "(WAL segment size change is not supported in 'DEFAULT' WAL mode) " +
+                        "[filePath=" + checkFile.getAbsolutePath() +
+                        ", fileSize=" + checkFile.length() +
+                        ", configSize=" + dsCfg.getWalSegments() + ']');
             }
             else if (create)
                 createFile(checkFile);
+
+            if (completionCallback != null)
+                completionCallback.apply(i);
         }
     }
 
@@ -2130,6 +2276,13 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
         public String getAbsolutePath() {
             return file.getAbsolutePath();
         }
+
+        /**
+         * @return True if segment is ZIP compressed.
+         */
+        public boolean isCompressed() {
+            return file.getName().endsWith(".zip");
+        }
     }
 
     /**
@@ -2444,7 +2597,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
                     if (!needFsync(ptr))
                         return;
 
-                    if (fsyncDelay > 0 && !this.stop.get()) {
+                    if (fsyncDelay > 0 && !stop.get()) {
                         // Delay fsync to collect as many updates as possible: trade latency for throughput.
                         U.await(fsync, fsyncDelay, TimeUnit.NANOSECONDS);
 
@@ -2455,7 +2608,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
 
                 flushOrWait(ptr);
 
-                if (this.stop.get())
+                if (stop.get())
                     return;
 
                 long lastFsyncPos0 = lastFsyncPos;
@@ -2537,9 +2690,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
          */
         private boolean close(boolean rollOver) throws IgniteCheckedException, StorageException {
             if (stop.compareAndSet(false, true)) {
-                try {
-                    lock.lock();
+                lock.lock();
 
+                try {
                     flushOrWait(null);
 
                     try {
@@ -2583,7 +2736,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
                             try {
                                 fileIO.close();
                             }
-                            catch (IOException e) {
+                            catch (IOException ignore) {
                                 // No-op.
                             }
                         }
@@ -2634,9 +2787,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
         }
 
         /**
-         * @throws IgniteCheckedException If failed.
+         *
          */
-        private void awaitNext() throws IgniteCheckedException {
+        private void awaitNext() {
             lock.lock();
 
             try {
@@ -2686,7 +2839,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
          * @return Safely reads current position of the file channel as String. Will return "null" if channel is null.
          */
         private String safePosition() {
-            FileIO io = this.fileIO;
+            FileIO io = fileIO;
 
             if (io == null)
                 return "null";
@@ -2854,7 +3007,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
 
         /** {@inheritDoc} */
         @Override protected ReadFileHandle advanceSegment(
-            @Nullable final ReadFileHandle curWalSegment) throws IgniteCheckedException {
+            @Nullable final ReadFileHandle curWalSegment
+        ) throws IgniteCheckedException {
             if (curWalSegment != null) {
                 curWalSegment.close();
 
@@ -2905,7 +3059,6 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
             else
                 nextHandle.workDir = !readArchive;
 
-
             curRec = null;
 
             return nextHandle;
@@ -2947,7 +3100,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     /**
      * WAL writer worker.
      */
-    class WALWriter extends Thread {
+    @SuppressWarnings("ForLoopReplaceableByForEach")
+    private class WALWriter extends Thread {
         /** Unconditional flush. */
         private static final long UNCONDITIONAL_FLUSH = -1L;
 


[13/17] ignite git commit: IEP-4 Baseline topology for persistent caches (Phase 1) Contributed by: Dmitriy Govorukhin Dmitry Pavlov Eduard Shangareev Ily

Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 6e4b726..3599594 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -77,6 +77,8 @@ import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
+import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
@@ -91,6 +93,7 @@ import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
 import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
+import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
 import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
 import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
@@ -108,6 +111,8 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.Gri
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.CheckpointMetricsTracker;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl;
@@ -140,6 +145,7 @@ import org.apache.ignite.internal.util.worker.GridWorker;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteOutClosure;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.mxbean.DataStorageMetricsMXBean;
 import org.apache.ignite.thread.IgniteThread;
 import org.apache.ignite.thread.IgniteThreadPoolExecutor;
@@ -147,8 +153,10 @@ import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 import static java.nio.file.StandardOpenOption.READ;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
+import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_ID;
 
 /**
  *
@@ -158,6 +166,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** */
     public static final String IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC = "IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC";
 
+    /** MemoryPolicyConfiguration name reserved for meta store. */
+    private static final String METASTORE_DATA_REGION_NAME = "metastoreMemPlc";
+
     /** */
     private static final long GB = 1024L * 1024 * 1024;
 
@@ -190,6 +201,12 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** Checkpoint file name pattern. */
     private static final Pattern CP_FILE_NAME_PATTERN = Pattern.compile("(\\d+)-(.*)-(START|END)\\.bin");
 
+    /** Node started file patter. */
+    private static final Pattern NODE_STARTED_FILE_NAME_PATTERN = Pattern.compile("(\\d+)-node-started\\.bin");
+
+    /** Node started file suffix. */
+    private static final String NODE_STARTED_FILE_NAME_SUFFIX = "-node-started.bin";
+
     /** */
     private static final FileFilter CP_FILE_FILTER = new FileFilter() {
         @Override public boolean accept(File f) {
@@ -198,6 +215,13 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     };
 
     /** */
+    private static final FileFilter NODE_STARTED_FILE_FILTER = new FileFilter() {
+        @Override public boolean accept(File f) {
+            return f.getName().endsWith(NODE_STARTED_FILE_NAME_SUFFIX);
+        }
+    };
+
+    /** */
     private static final Comparator<GridDhtLocalPartition> ASC_PART_COMPARATOR = new Comparator<GridDhtLocalPartition>() {
         @Override public int compare(GridDhtLocalPartition a, GridDhtLocalPartition b) {
             return Integer.compare(a.id(), b.id());
@@ -296,6 +320,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     private final long lockWaitTime;
 
     /** */
+    private final int maxCpHistMemSize;
+
+    /** */
     private Map<Integer, Map<Integer, T2<Long, WALPointer>>> reservedForExchange;
 
     /** */
@@ -316,6 +343,12 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** Number of pages in current checkpoint. */
     private volatile int currCheckpointPagesCnt;
 
+    /** */
+    private MetaStorage metaStorage;
+
+    /** */
+    private List<MetastorageLifecycleListener> metastorageLifecycleLsnrs;
+
     /**
      * @param ctx Kernal context.
      */
@@ -335,6 +368,23 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
             persistenceCfg.getMetricsRateTimeInterval(),
             persistenceCfg.getMetricsSubIntervalCount()
         );
+
+        metastorageLifecycleLsnrs = ctx.internalSubscriptionProcessor().getMetastorageSubscribers();
+
+        maxCpHistMemSize = Math.min(persistenceCfg.getWalHistorySize(),
+            IgniteSystemProperties.getInteger(IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE, 100));
+    }
+
+    /** */
+    private void notifyMetastorageReadyForRead() throws IgniteCheckedException {
+        for (MetastorageLifecycleListener lsnr : metastorageLifecycleLsnrs)
+            lsnr.onReadyForRead(metaStorage);
+    }
+
+    /** */
+    private void notifyMetastorageReadyForReadWrite() throws IgniteCheckedException {
+        for (MetastorageLifecycleListener lsnr : metastorageLifecycleLsnrs)
+            lsnr.onReadyForReadWrite(metaStorage);
     }
 
     /**
@@ -360,6 +410,31 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     }
 
     /** {@inheritDoc} */
+    @Override protected void initDataRegions(DataStorageConfiguration memCfg) throws IgniteCheckedException {
+        super.initDataRegions(memCfg);
+
+        addDataRegion(
+            memCfg,
+            createDataRegionConfiguration(memCfg),
+            false
+        );
+    }
+
+    /**
+     * @param storageCfg Data storage configuration.
+     * @return Data region configuration.
+     */
+    private DataRegionConfiguration createDataRegionConfiguration(DataStorageConfiguration storageCfg) {
+        DataRegionConfiguration cfg = new DataRegionConfiguration();
+
+        cfg.setName(METASTORE_DATA_REGION_NAME);
+        cfg.setInitialSize(storageCfg.getSystemRegionInitialSize());
+        cfg.setMaxSize(storageCfg.getSystemRegionMaxSize());
+        cfg.setPersistenceEnabled(true);
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
     @Override protected void start0() throws IgniteCheckedException {
         super.start0();
 
@@ -397,6 +472,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
                 fileLockHolder = new FileLockHolder(storeMgr.workDir().getPath(), kernalCtx, log);
 
             persStoreMetrics.wal(cctx.wal());
+
+            // Here we can get data from metastorage
+            readMetastore();
         }
     }
 
@@ -415,6 +493,59 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
             );
     }
 
+    /** */
+    private void readMetastore() throws IgniteCheckedException {
+        try {
+            DataStorageConfiguration memCfg = cctx.kernalContext().config().getDataStorageConfiguration();
+
+            DataRegionConfiguration plcCfg = createDataRegionConfiguration(memCfg);
+
+            File allocPath = buildAllocPath(plcCfg);
+
+            DirectMemoryProvider memProvider = allocPath == null ?
+                new UnsafeMemoryProvider(log) :
+                new MappedFileMemoryProvider(
+                    log,
+                    allocPath);
+
+            DataRegionMetricsImpl memMetrics = new DataRegionMetricsImpl(plcCfg);
+
+            PageMemoryEx storePageMem = (PageMemoryEx)createPageMemory(memProvider, memCfg, plcCfg, memMetrics, false);
+
+            DataRegion regCfg = new DataRegion(storePageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg, storePageMem));
+
+            CheckpointStatus status = readCheckpointStatus();
+
+            cctx.pageStore().initializeForMetastorage();
+
+            storePageMem.start();
+
+            checkpointReadLock();
+
+            try {
+                restoreMemory(status, true, storePageMem);
+
+                metaStorage = new MetaStorage(cctx.wal(), regCfg, memMetrics, true);
+
+                metaStorage.init(this);
+
+                applyLastUpdates(status, true);
+
+                notifyMetastorageReadyForRead();
+            }
+            finally {
+                checkpointReadUnlock();
+            }
+
+            metaStorage = null;
+
+            storePageMem.stop();
+        }
+        catch (StorageException e) {
+            throw new IgniteCheckedException(e);
+        }
+    }
+
     /**
      * Get checkpoint buffer size for the given configuration.
      *
@@ -552,7 +683,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
 
     /** {@inheritDoc} */
     @Override public void readCheckpointAndRestoreMemory(
-        List<DynamicCacheDescriptor> cachesToStart) throws IgniteCheckedException {
+        List<DynamicCacheDescriptor> cachesToStart
+    ) throws IgniteCheckedException {
+        assert !cctx.localNode().isClient();
+
         checkpointReadLock();
 
         try {
@@ -565,13 +699,29 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
 
             CheckpointStatus status = readCheckpointStatus();
 
+            cctx.pageStore().initializeForMetastorage();
+
+            metaStorage = new MetaStorage(cctx.wal(), dataRegionMap.get(METASTORE_DATA_REGION_NAME),
+                (DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME));
+
+            WALPointer restore = restoreMemory(status);
+
             // First, bring memory to the last consistent checkpoint state if needed.
             // This method should return a pointer to the last valid record in the WAL.
-            WALPointer restore = restoreMemory(status);
 
             cctx.wal().resumeLogging(restore);
 
-            cctx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
+            WALPointer ptr = cctx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
+
+            if (ptr != null) {
+                cctx.wal().fsync(ptr);
+
+                nodeStart(ptr);
+            }
+
+            metaStorage.init(this);
+
+            notifyMetastorageReadyForReadWrite();
         }
         catch (StorageException e) {
             throw new IgniteCheckedException(e);
@@ -581,6 +731,93 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
         }
     }
 
+    /**
+     * @param ptr Memory recovery wal pointer.
+     */
+    private void nodeStart(WALPointer ptr) throws IgniteCheckedException {
+        FileWALPointer p = (FileWALPointer)ptr;
+
+        String fileName = U.currentTimeMillis() + "-node-started.bin";
+
+        ByteBuffer buf = ByteBuffer.allocate(20);
+        buf.order(ByteOrder.nativeOrder());
+
+        try (FileChannel ch = FileChannel.open(
+            Paths.get(cpDir.getAbsolutePath(), fileName),
+            StandardOpenOption.CREATE_NEW, StandardOpenOption.APPEND)
+        ) {
+            buf.putLong(p.index());
+
+            buf.putInt(p.fileOffset());
+
+            buf.putInt(p.length());
+
+            buf.flip();
+
+            ch.write(buf);
+
+            buf.clear();
+
+            ch.force(true);
+        }
+        catch (IOException e) {
+            throw new IgniteCheckedException(e);
+        }
+    }
+
+    /**
+     *
+     */
+    public List<T2<Long, WALPointer>> nodeStartedPointers() throws IgniteCheckedException {
+        List<T2<Long, WALPointer>> res = new ArrayList<>();
+
+        File[] files = cpDir.listFiles(NODE_STARTED_FILE_FILTER);
+
+        Arrays.sort(files, new Comparator<File>() {
+            @Override public int compare(File o1, File o2) {
+                String n1 = o1.getName();
+                String n2 = o2.getName();
+
+                Long ts1 = Long.valueOf(n1.substring(0, n1.length() - NODE_STARTED_FILE_NAME_SUFFIX.length()));
+                Long ts2 = Long.valueOf(n2.substring(0, n2.length() - NODE_STARTED_FILE_NAME_SUFFIX.length()));
+
+                if (ts1 == ts2)
+                    return 0;
+                else if (ts1 < ts2)
+                    return -1;
+                else
+                    return 1;
+            }
+        });
+
+        ByteBuffer buf = ByteBuffer.allocate(20);
+        buf.order(ByteOrder.nativeOrder());
+
+        for (File f : files){
+            String name = f.getName();
+
+            Long ts = Long.valueOf(name.substring(0, name.length() - NODE_STARTED_FILE_NAME_SUFFIX.length()));
+
+            try (FileChannel ch = FileChannel.open(f.toPath(), READ)) {
+                ch.read(buf);
+
+                buf.flip();
+
+                FileWALPointer ptr = new FileWALPointer(
+                    buf.getLong(), buf.getInt(), buf.getInt());
+
+                res.add(new T2<Long, WALPointer>(ts, ptr));
+
+                buf.clear();
+            }
+            catch (IOException e) {
+                throw new IgniteCheckedException("Failed to read node started marker file: " + f.getAbsolutePath(), e);
+            }
+        }
+
+        return res;
+    }
+
     /** {@inheritDoc} */
     @Override public void lock() throws IgniteCheckedException {
         if (fileLockHolder != null) {
@@ -655,10 +892,11 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
         DirectMemoryProvider memProvider,
         DataStorageConfiguration memCfg,
         DataRegionConfiguration plcCfg,
-        DataRegionMetricsImpl memMetrics
+        DataRegionMetricsImpl memMetrics,
+        final boolean trackable
     ) {
         if (!plcCfg.isPersistenceEnabled())
-            return super.createPageMemory(memProvider, memCfg, plcCfg, memMetrics);
+            return super.createPageMemory(memProvider, memCfg, plcCfg, memMetrics, trackable);
 
         memMetrics.persistenceEnabled(true);
 
@@ -680,6 +918,22 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
         if (IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_OVERRIDE_WRITE_THROTTLING_ENABLED, false))
             writeThrottlingEnabled = true;
 
+        GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker;
+
+        if (trackable)
+            changeTracker = new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
+                @Override public void applyx(
+                    Long page,
+                    FullPageId fullId,
+                    PageMemoryEx pageMem
+                ) throws IgniteCheckedException {
+                    if (trackable)
+                        snapshotMgr.onChangeTrackerPage(page, fullId, pageMem);
+                }
+            };
+        else
+            changeTracker = null;
+
         PageMemoryImpl pageMem = new PageMemoryImpl(
             memProvider,
             calculateFragmentSizes(
@@ -702,15 +956,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
                     snapshotMgr.flushDirtyPageHandler(fullId, pageBuf, tag);
                 }
             },
-            new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
-                @Override public void applyx(
-                    Long page,
-                    FullPageId fullId,
-                    PageMemoryEx pageMem
-                ) throws IgniteCheckedException {
-                    snapshotMgr.onChangeTrackerPage(page, fullId, pageMem);
-                }
-            },
+            changeTracker,
             this,
             memMetrics,
             writeThrottlingEnabled
@@ -1071,7 +1317,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
             checkpointReadLock();
 
             try {
-                applyLastUpdates(status);
+                applyLastUpdates(status, false);
             }
             finally {
                 checkpointReadUnlock();
@@ -1082,6 +1328,11 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
             checkpointer = new Checkpointer(cctx.igniteInstanceName(), "db-checkpoint-thread", log);
 
             new IgniteThread(cctx.igniteInstanceName(), "db-checkpoint-thread", checkpointer).start();
+
+            CheckpointProgressSnapshot chp = checkpointer.wakeupForCheckpoint(0, "node started");
+
+            if (chp != null)
+                chp.cpBeginFut.get();
         }
         catch (StorageException e) {
             throw new IgniteCheckedException(e);
@@ -1440,6 +1691,17 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
      * @param status Checkpoint status.
      */
     private WALPointer restoreMemory(CheckpointStatus status) throws IgniteCheckedException {
+        return restoreMemory(status, false, (PageMemoryEx)metaStorage.pageMemory());
+    }
+
+    /**
+     * @param status Checkpoint status.
+     * @param storeOnly If {@code True} restores Metastorage only.
+     */
+    private WALPointer restoreMemory(CheckpointStatus status, boolean storeOnly,
+        PageMemoryEx storePageMem) throws IgniteCheckedException {
+        assert !storeOnly || storePageMem != null;
+
         if (log.isInfoEnabled())
             log.info("Checking memory state [lastValidPos=" + status.endPtr + ", lastMarked="
                 + status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']');
@@ -1491,9 +1753,13 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
                             // Here we do not require tag check because we may be applying memory changes after
                             // several repetitive restarts and the same pages may have changed several times.
                             int grpId = pageRec.fullPageId().groupId();
+
+                            if (storeOnly && grpId != METASTORAGE_CACHE_ID)
+                                continue;
+
                             long pageId = pageRec.fullPageId().pageId();
 
-                            PageMemoryEx pageMem = getPageMemoryForCacheGroup(grpId);
+                            PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
 
                             long page = pageMem.acquirePage(grpId, pageId, true);
 
@@ -1521,9 +1787,13 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
                             PartitionDestroyRecord destroyRec = (PartitionDestroyRecord)rec;
 
                             final int gId = destroyRec.groupId();
+
+                            if (storeOnly && gId != METASTORAGE_CACHE_ID)
+                                continue;
+
                             final int pId = destroyRec.partitionId();
 
-                            PageMemoryEx pageMem = getPageMemoryForCacheGroup(gId);
+                            PageMemoryEx pageMem = gId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(gId);
 
                             pageMem.clearAsync(new P3<Integer, Long, Integer>() {
                                 @Override public boolean apply(Integer cacheId, Long pageId, Integer tag) {
@@ -1539,9 +1809,13 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
                             PageDeltaRecord r = (PageDeltaRecord)rec;
 
                             int grpId = r.groupId();
+
+                            if (storeOnly && grpId != METASTORAGE_CACHE_ID)
+                                continue;
+
                             long pageId = r.pageId();
 
-                            PageMemoryEx pageMem = getPageMemoryForCacheGroup(grpId);
+                            PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
 
                             // Here we do not require tag check because we may be applying memory changes after
                             // several repetitive restarts and the same pages may have changed several times.
@@ -1567,6 +1841,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
             }
         }
 
+        if (storeOnly)
+            return null;
+
         if (status.needRestoreMemory()) {
             if (apply)
                 throw new IgniteCheckedException("Failed to restore memory state (checkpoint marker is present " +
@@ -1607,16 +1884,87 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     }
 
     /**
+     * Apply update from some iterator and with specific filters.
+     *
+     * @param it WalIterator.
+     * @param recPredicate Wal record filter.
+     * @param entryPredicate Entry filter.
+     * @param partStates Partition to restore state.
+     */
+    public void applyUpdatesOnRecovery(
+        WALIterator it,
+        IgnitePredicate<IgniteBiTuple<WALPointer, WALRecord>> recPredicate,
+        IgnitePredicate<DataEntry> entryPredicate,
+        Map<T2<Integer, Integer>, T2<Integer, Long>> partStates
+    ) throws IgniteCheckedException {
+        while (it.hasNextX()) {
+            IgniteBiTuple<WALPointer, WALRecord> next = it.nextX();
+
+            WALRecord rec = next.get2();
+
+            if (!recPredicate.apply(next))
+                break;
+
+            switch (rec.type()) {
+                case DATA_RECORD:
+                    checkpointReadLock();
+
+                    try {
+                        DataRecord dataRec = (DataRecord) rec;
+
+                        for (DataEntry dataEntry : dataRec.writeEntries()) {
+                            if (entryPredicate.apply(dataEntry)) {
+                                checkpointReadLock();
+
+                                try {
+                                    int cacheId = dataEntry.cacheId();
+
+                                    GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
+
+                                    if (cacheCtx != null)
+                                        applyUpdate(cacheCtx, dataEntry);
+                                    else if (log != null)
+                                        log.warning("Cache (cacheId=" + cacheId + ") is not started, can't apply updates.");
+                                }
+                                finally {
+                                    checkpointReadUnlock();
+                                }
+                            }
+                        }
+                    }
+                    finally {
+                        checkpointReadUnlock();
+                    }
+
+                    break;
+
+                default:
+                    // Skip other records.
+            }
+        }
+
+        checkpointReadLock();
+
+        try {
+            restorePartitionState(partStates);
+        }
+        finally {
+            checkpointReadUnlock();
+        }
+    }
+
+    /**
      * @param status Last registered checkpoint status.
      * @throws IgniteCheckedException If failed to apply updates.
      * @throws StorageException If IO exception occurred while reading write-ahead log.
      */
-    private void applyLastUpdates(CheckpointStatus status) throws IgniteCheckedException {
+    private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) throws IgniteCheckedException {
         if (log.isInfoEnabled())
             log.info("Applying lost cache updates since last checkpoint record [lastMarked="
                 + status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']');
 
-        cctx.kernalContext().query().skipFieldLookup(true);
+        if (!metastoreOnly)
+            cctx.kernalContext().query().skipFieldLookup(true);
 
         long start = U.currentTimeMillis();
         int applied = 0;
@@ -1631,6 +1979,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
 
                 switch (rec.type()) {
                     case DATA_RECORD:
+                        if (metastoreOnly)
+                            continue;
+
                         DataRecord dataRec = (DataRecord)rec;
 
                         for (DataEntry dataEntry : dataRec.writeEntries()) {
@@ -1646,6 +1997,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
                         break;
 
                     case PART_META_UPDATE_STATE:
+                        if (metastoreOnly)
+                            continue;
+
                         PartitionMetaStateRecord metaStateRecord = (PartitionMetaStateRecord)rec;
 
                         partStates.put(new T2<>(metaStateRecord.groupId(), metaStateRecord.partitionId()),
@@ -1653,15 +2007,52 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
 
                         break;
 
+                    case METASTORE_DATA_RECORD:
+                        MetastoreDataRecord metastoreDataRecord = (MetastoreDataRecord)rec;
+
+                        metaStorage.applyUpdate(metastoreDataRecord.key(), metastoreDataRecord.value());
+
+                        break;
+
+                    case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID:
+                    case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID:
+                    case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID:
+                        if (metastoreOnly)
+                            continue;
+
+                        PageDeltaRecord rec0 = (PageDeltaRecord) rec;
+
+                        PageMemoryEx pageMem = getPageMemoryForCacheGroup(rec0.groupId());
+
+                        long page = pageMem.acquirePage(rec0.groupId(), rec0.pageId(), true);
+
+                        try {
+                            long addr = pageMem.writeLock(rec0.groupId(), rec0.pageId(), page, true);
+
+                            try {
+                                rec0.applyDelta(pageMem, addr);
+                            }
+                            finally {
+                                pageMem.writeUnlock(rec0.groupId(), rec0.pageId(), page, null, true, true);
+                            }
+                        }
+                        finally {
+                            pageMem.releasePage(rec0.groupId(), rec0.pageId(), page);
+                        }
+
+                        break;
+
                     default:
                         // Skip other records.
                 }
             }
 
-            restorePartitionState(partStates);
+            if (!metastoreOnly)
+                restorePartitionState(partStates);
         }
         finally {
-            cctx.kernalContext().query().skipFieldLookup(false);
+            if (!metastoreOnly)
+                cctx.kernalContext().query().skipFieldLookup(false);
         }
 
         if (log.isInfoEnabled())
@@ -1690,6 +2081,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
             PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
 
             for (int i = 0; i < grp.affinity().partitions(); i++) {
+                T2<Integer, Long> restore = partStates.get(new T2<>(grpId, i));
+
                 if (storeMgr.exists(grpId, i)) {
                     storeMgr.ensure(grpId, i);
 
@@ -1703,41 +2096,99 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
                     // TODO: https://issues.apache.org/jira/browse/IGNITE-6097
                     grp.offheap().onPartitionInitialCounterUpdated(i, 0);
 
-                    long partMetaId = pageMem.partitionMetaPageId(grpId, i);
-                    long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
+                    checkpointReadLock();
 
                     try {
-                        long pageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage);
-
-                        boolean changed = false;
+                        long partMetaId = pageMem.partitionMetaPageId(grpId, i);
+                        long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
 
                         try {
-                            PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.forPage(pageAddr);
+                            long pageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage);
 
-                            T2<Integer, Long> fromWal = partStates.get(new T2<>(grpId, i));
+                            boolean changed = false;
 
-                            if (fromWal != null) {
-                                int stateId = fromWal.get1();
+                            try {
+                                PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.forPage(pageAddr);
+
+                                if (restore != null) {
+                                    int stateId = restore.get1();
 
-                                io.setPartitionState(pageAddr, (byte)stateId);
+                                    io.setPartitionState(pageAddr, (byte)stateId);
 
-                                changed = updateState(part, stateId);
+                                    changed = updateState(part, stateId);
 
-                                if (stateId == GridDhtPartitionState.OWNING.ordinal()) {
-                                    grp.offheap().onPartitionInitialCounterUpdated(i, fromWal.get2());
+                                    if (stateId == GridDhtPartitionState.OWNING.ordinal()
+                                        || (stateId == GridDhtPartitionState.MOVING.ordinal()
 
-                                    if (part.initialUpdateCounter() < fromWal.get2()) {
-                                        part.initialUpdateCounter(fromWal.get2());
+                                        && part.initialUpdateCounter() < restore.get2())) {
+                                        part.initialUpdateCounter(restore.get2());
 
                                         changed = true;
                                     }
                                 }
+                                else
+                                    updateState(part, (int)io.getPartitionState(pageAddr));
+                            }
+                            finally {
+                                pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, changed);
+                            }
+                        }
+                        finally {
+                            pageMem.releasePage(grpId, partMetaId, partMetaPage);
+                        }
+                    }
+                    finally {
+                        checkpointReadUnlock();
+                    }
+                }
+                else if (restore != null) {
+                    GridDhtLocalPartition part = grp.topology().forceCreatePartition(i);
+
+                    assert part != null;
+
+                    // TODO: https://issues.apache.org/jira/browse/IGNITE-6097
+                    grp.offheap().onPartitionInitialCounterUpdated(i, 0);
+
+                    updateState(part, restore.get1());
+                }
+            }
+        }
+    }
+
+    /**
+     * @param grpCtx Group context.
+     * @param partId Partition ID.
+     * @return Partition state.
+     */
+    public GridDhtPartitionState readPartitionState(CacheGroupContext grpCtx, int partId) {
+        int grpId = grpCtx.groupId();
+        PageMemoryEx pageMem = (PageMemoryEx)grpCtx.dataRegion().pageMemory();
+
+        try {
+            if (storeMgr.exists(grpId, partId)) {
+                storeMgr.ensure(grpId, partId);
+
+                if (storeMgr.pages(grpId, partId) > 1) {
+                    long partMetaId = pageMem.partitionMetaPageId(grpId, partId);
+                    long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
+
+                    try {
+                        long pageAddr = pageMem.readLock(grpId, partMetaId, partMetaPage);
+
+                        try {
+                            if (PageIO.getType(pageAddr) == PageIO.T_PART_META) {
+                                PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.forPage(pageAddr);
+
+                                GridDhtPartitionState state = GridDhtPartitionState.fromOrdinal((int)io.getPartitionState(pageAddr));
+
+                                if (state == null)
+                                    state = GridDhtPartitionState.MOVING;
+
+                                return state;
                             }
-                            else
-                                changed = updateState(part, (int)io.getPartitionState(pageAddr));
                         }
                         finally {
-                            pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, changed);
+                            pageMem.readUnlock(grpId, partMetaId, partMetaPage);
                         }
                     }
                     finally {
@@ -1746,6 +2197,21 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
                 }
             }
         }
+        catch (IgniteCheckedException e) {
+            U.error(log, "Failed to read partition state (will default to MOVING) [grp=" + grpCtx +
+                ", partId=" + partId + "]", e);
+        }
+
+        return GridDhtPartitionState.MOVING;
+    }
+
+    /**
+     * Wal truncate callBack.
+     *
+     * @param highBound WALPointer.
+     */
+    public void onWalTruncated(WALPointer highBound) {
+        checkpointHist.onWalTruncated(highBound);
     }
 
     /**
@@ -1805,6 +2271,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
 
                 break;
 
+            case READ:
+                // do nothing
+                break;
+
             default:
                 throw new IgniteCheckedException("Invalid operation for WAL entry update: " + dataEntry.op());
         }
@@ -2330,8 +2800,14 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
 
                     CacheState state = new CacheState(locParts.size());
 
-                    for (GridDhtLocalPartition part : grp.topology().currentLocalPartitions())
-                        state.addPartitionState(part.id(), part.dataStore().fullSize(), part.updateCounter());
+                    for (GridDhtLocalPartition part : grp.topology().currentLocalPartitions()) {
+                        state.addPartitionState(
+                            part.id(),
+                            part.dataStore().fullSize(),
+                            part.updateCounter(),
+                            (byte)part.state().ordinal()
+                        );
+                    }
 
                     cpRec.addCacheGroupState(grp.groupId(), state);
                 }
@@ -2340,7 +2816,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
 
                 hasPages = hasPageForWrite(cpPagesTuple.get1());
 
-                if (hasPages) {
+                if (hasPages || curr.nextSnapshot) {
                     // No page updates for this checkpoint are allowed from now on.
                     cpPtr = cctx.wal().log(cpRec);
 
@@ -2631,15 +3107,22 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
 
                     int grpId = fullId.groupId();
 
-                    CacheGroupContext grp = context().cache().cacheGroup(grpId);
+                    PageMemoryEx pageMem;
 
-                    if (grp == null)
-                        continue;
+                    if (grpId != MetaStorage.METASTORAGE_CACHE_ID) {
+                        CacheGroupContext grp = context().cache().cacheGroup(grpId);
 
-                    if (!grp.dataRegion().config().isPersistenceEnabled())
-                        continue;
+                        if (grp == null)
+                            continue;
+
+                        if (!grp.dataRegion().config().isPersistenceEnabled())
+                            continue;
+
+                        pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
+                    }
+                    else
+                        pageMem = (PageMemoryEx)metaStorage.pageMemory();
 
-                    PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
 
                     Integer tag = pageMem.getForCheckpoint(
                         fullId, tmpWriteBuf, persStoreMetrics.metricsEnabled() ? tracker : null);
@@ -2935,15 +3418,18 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
         }
 
         /**
-         * Clears checkpoint history.
+         * Callback on truncate wal.
          */
-        private void onCheckpointFinished(Checkpoint chp) {
-            int deleted = 0;
+        private void onWalTruncated(WALPointer ptr) {
+            FileWALPointer highBound = (FileWALPointer)ptr;
 
-            while (histMap.size() > persistenceCfg.getWalHistorySize()) {
-                Map.Entry<Long, CheckpointEntry> entry = histMap.firstEntry();
+            List<CheckpointEntry> cpToRemove = new ArrayList<>();
 
-                CheckpointEntry cpEntry = entry.getValue();
+            for (CheckpointEntry cpEntry : histMap.values()) {
+                FileWALPointer cpPnt = (FileWALPointer)cpEntry.checkpointMark();
+
+                if (highBound.compareTo(cpPnt) <= 0)
+                    break;
 
                 if (cctx.wal().reserved(cpEntry.checkpointMark())) {
                     U.warn(log, "Could not clear historyMap due to WAL reservation on cpEntry " + cpEntry.cpId +
@@ -2952,27 +3438,39 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
                     break;
                 }
 
-                File startFile = new File(cpDir.getAbsolutePath(), cpEntry.startFile());
-                File endFile = new File(cpDir.getAbsolutePath(), cpEntry.endFile());
+                if (!removeCheckpointFiles(cpEntry))
+                    cpToRemove.add(cpEntry);
+            }
+
+            for (CheckpointEntry cpEntry : cpToRemove)
+                histMap.remove(cpEntry.cpTs);
+        }
 
-                boolean rmvdStart = !startFile.exists() || startFile.delete();
-                boolean rmvdEnd = !endFile.exists() || endFile.delete();
+        /**
+         * Clears checkpoint history.
+         */
+        private void onCheckpointFinished(Checkpoint chp) {
+            int deleted = 0;
 
-                boolean fail = !rmvdStart || !rmvdEnd;
+            boolean dropWal = persistenceCfg.getWalHistorySize() != Integer.MAX_VALUE;
 
-                if (fail) {
-                    U.warn(log, "Failed to remove stale checkpoint files [startFile=" + startFile.getAbsolutePath() +
-                        ", endFile=" + endFile.getAbsolutePath() + ']');
+            while (histMap.size() > maxCpHistMemSize) {
+                Map.Entry<Long, CheckpointEntry> entry = histMap.firstEntry();
 
-                    if (histMap.size() > 2 * persistenceCfg.getWalHistorySize()) {
-                        U.error(log, "Too many stale checkpoint entries in the map, will truncate WAL archive anyway.");
+                CheckpointEntry cpEntry = entry.getValue();
 
-                        fail = false;
-                    }
+                if (cctx.wal().reserved(cpEntry.checkpointMark())) {
+                    U.warn(log, "Could not clear historyMap due to WAL reservation on cpEntry " + cpEntry.cpId +
+                        ", history map size is " + histMap.size());
+
+                    break;
                 }
 
+                boolean fail = removeCheckpointFiles(cpEntry);
+
                 if (!fail) {
-                    deleted += cctx.wal().truncate(cpEntry.checkpointMark());
+                    if (dropWal)
+                        deleted += cctx.wal().truncate(null, cpEntry.checkpointMark());
 
                     histMap.remove(entry.getKey());
                 }
@@ -2987,6 +3485,33 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
         }
 
         /**
+         * @param cpEntry Checkpoint entry.
+         * @return {True} if delete fail.
+         */
+        private boolean removeCheckpointFiles(CheckpointEntry cpEntry) {
+            File startFile = new File(cpDir.getAbsolutePath(), cpEntry.startFile());
+            File endFile = new File(cpDir.getAbsolutePath(), cpEntry.endFile());
+
+            boolean rmvdStart = !startFile.exists() || startFile.delete();
+            boolean rmvdEnd = !endFile.exists() || endFile.delete();
+
+            boolean fail = !rmvdStart || !rmvdEnd;
+
+            if (fail) {
+                U.warn(log, "Failed to remove stale checkpoint files [startFile=" + startFile.getAbsolutePath() +
+                    ", endFile=" + endFile.getAbsolutePath() + ']');
+
+                if (histMap.size() > 2 * maxCpHistMemSize) {
+                    U.error(log, "Too many stale checkpoint entries in the map, will truncate WAL archive anyway.");
+
+                    fail = false;
+                }
+            }
+
+            return fail;
+        }
+
+        /**
          * @param cacheId Cache ID.
          * @param partId Partition ID.
          * @return Reserved counter or null if couldn't reserve.
@@ -3360,4 +3885,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     public DataStorageMetricsImpl persistentStoreMetricsImpl() {
         return persStoreMetrics;
     }
-}
\ No newline at end of file
+
+    /** {@inheritDoc} */
+    @Override public MetaStorage metaStorage() {
+        return metaStorage;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 5ccbfb9..f8fd86c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -30,6 +30,7 @@ import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageSupport;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
 import org.apache.ignite.internal.pagemem.wal.WALIterator;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
@@ -49,8 +50,7 @@ import org.apache.ignite.internal.processors.cache.IgniteRebalanceIterator;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
-import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeListImpl;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
 import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId;
 import org.apache.ignite.internal.processors.cache.persistence.partstate.PagesAllocationRange;
@@ -67,7 +67,6 @@ import org.apache.ignite.internal.processors.cache.tree.CacheDataTree;
 import org.apache.ignite.internal.processors.cache.tree.PendingEntriesTree;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.query.GridQueryRowCacheCleaner;
-import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.lang.GridCursor;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.T2;
@@ -76,12 +75,16 @@ import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.RENTING;
+
 /**
  * Used when persistence enabled.
  */
 public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl implements DbCheckpointListener {
     /** */
-    private MetaStore metaStore;
+    private IndexStorage indexStorage;
 
     /** */
     private ReuseListImpl reuseList;
@@ -101,7 +104,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
 
         RootPage metastoreRoot = metas.treeRoot;
 
-        metaStore = new MetadataStorage(grp.dataRegion().pageMemory(),
+        indexStorage = new IndexStorageImpl(grp.dataRegion().pageMemory(),
             ctx.wal(),
             globalRemoveId(),
             grp.groupId(),
@@ -122,7 +125,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
             try {
                 final String name = "PendingEntries";
 
-                RootPage pendingRootPage = metaStore.getOrAllocateForTree(name);
+                RootPage pendingRootPage = indexStorage.getOrAllocateForTree(name);
 
                 pendingEntries = new PendingEntriesTree(
                     grp,
@@ -169,8 +172,12 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
      * @param store Store to save metadata.
      * @throws IgniteCheckedException If failed.
      */
-    private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean saveMeta,
-        boolean beforeDestroy) throws IgniteCheckedException {
+    private boolean saveStoreMetadata(
+        CacheDataStore store,
+        Context ctx,
+        boolean saveMeta,
+        boolean beforeDestroy
+    ) throws IgniteCheckedException {
         RowStore rowStore0 = store.rowStore();
 
         boolean needSnapshot = ctx != null && ctx.nextSnapshot() && ctx.needToSnapshot(grp.cacheOrGroupName());
@@ -178,7 +185,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
         boolean wasSaveToMeta = false;
 
         if (rowStore0 != null) {
-            FreeListImpl freeList = (FreeListImpl)rowStore0.freeList();
+            CacheFreeListImpl freeList = (CacheFreeListImpl)rowStore0.freeList();
 
             freeList.saveMetadata();
 
@@ -192,13 +199,15 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
             if (size > 0 || updCntr > 0) {
                 GridDhtPartitionState state = null;
 
+                // localPartition will not acquire writeLock here because create=false.
+                GridDhtLocalPartition part = null;
+
                 if (!grp.isLocal()) {
                     if (beforeDestroy)
                         state = GridDhtPartitionState.EVICTED;
                     else {
-                        // localPartition will not acquire writeLock here because create=false.
-                        GridDhtLocalPartition part = grp.topology().localPartition(store.partId(),
-                            AffinityTopologyVersion.NONE, false, true);
+                        part = grp.topology().localPartition(store.partId(),
+                                AffinityTopologyVersion.NONE, false, true);
 
                         if (part != null && part.state() != GridDhtPartitionState.EVICTED)
                             state = part.state();
@@ -241,63 +250,21 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
                         long cntrsPageId;
 
                         if (grp.sharedGroup()) {
-                            cntrsPageId = io.getCountersPageId(partMetaPageAddr);
-
-                            byte[] data = serializeCacheSizes(store.cacheSizes());
-
-                            int items = data.length / 12;
-                            int written = 0;
-                            int pageSize = pageMem.pageSize();
-
-                            boolean init = cntrsPageId == 0;
-
-                            if (init && items > 0) {
-                                cntrsPageId = pageMem.allocatePage(grpId, store.partId(), PageIdAllocator.FLAG_DATA);
-
-                                io.setCountersPageId(partMetaPageAddr, cntrsPageId);
-
-                                changed = true;
-                            }
-
-                            long nextId = cntrsPageId;
-
-                            while (written != items) {
-                                final long curId = nextId;
-                                final long curPage = pageMem.acquirePage(grpId, curId);
-
-                                try {
-                                    final long curAddr = pageMem.writeLock(grpId, curId, curPage);
-
-                                    assert curAddr != 0;
-
-                                    try {
-                                        PagePartitionCountersIO partMetaIo;
+                            long initCntrPageId = io.getCountersPageId(partMetaPageAddr);
 
-                                        if (init) {
-                                            partMetaIo = PagePartitionCountersIO.VERSIONS.latest();
+                            Map<Integer, Long> newSizes = store.cacheSizes();
+                            Map<Integer, Long> prevSizes = readSharedGroupCacheSizes(pageMem, grpId, initCntrPageId);
 
-                                            partMetaIo.initNewPage(curAddr, curId, pageSize);
-                                        }
-                                        else
-                                            partMetaIo = PageIO.getPageIO(curAddr);
+                            if (prevSizes != null && prevSizes.equals(newSizes))
+                                cntrsPageId = initCntrPageId; // Preventing modification of sizes pages for store
+                            else {
+                                cntrsPageId = writeSharedGroupCacheSizes(pageMem, grpId, initCntrPageId,
+                                    store.partId(), newSizes);
 
-                                        written += partMetaIo.writeCacheSizes(pageSize, curAddr, data, written);
+                                if (initCntrPageId == 0 && cntrsPageId != 0) {
+                                    io.setCountersPageId(partMetaPageAddr, cntrsPageId);
 
-                                        nextId = partMetaIo.getNextCountersPageId(curAddr);
-
-                                        if(written != items && (init = nextId == 0)) {
-                                            //allocate new counters page
-                                            nextId = pageMem.allocatePage(grpId, store.partId(), PageIdAllocator.FLAG_DATA);
-                                            partMetaIo.setNextCountersPageId(curAddr, nextId);
-                                        }
-                                    }
-                                    finally {
-                                        // Write full page
-                                        pageMem.writeUnlock(grpId, curId, curPage, Boolean.TRUE, true);
-                                    }
-                                }
-                                finally {
-                                    pageMem.releasePage(grpId, curId, curPage);
+                                    changed = true;
                                 }
                             }
                         }
@@ -311,49 +278,35 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
                             io.setCandidatePageCount(partMetaPageAddr, pageCnt);
 
                             if (saveMeta) {
-                                long metaPageId = pageMem.metaPageId(grpId);
-                                long metaPage = pageMem.acquirePage(grpId, metaPageId);
-
-                                try {
-                                    long metaPageAddr = pageMem.writeLock(grpId, metaPageId, metaPage);
-
-                                    try {
-                                        PageMetaIO metaIo = PageMetaIO.getPageIO(metaPageAddr);
-
-                                        long nextSnapshotTag = metaIo.getNextSnapshotTag(metaPageAddr);
-
-                                        metaIo.setNextSnapshotTag(metaPageAddr, nextSnapshotTag + 1);
-
-                                        if (log != null && log.isDebugEnabled())
-                                            log.debug("Save next snapshot before checkpoint start for grId = " + grpId
-                                                + ", nextSnapshotTag = " + nextSnapshotTag);
-
-                                        if (PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, metaPageId,
-                                            metaPage, wal, null))
-                                            wal.log(new MetaPageUpdateNextSnapshotId(grpId, metaPageId,
-                                                nextSnapshotTag + 1));
-
-                                        if (state == GridDhtPartitionState.OWNING)
-                                            addPartition(ctx.partitionStatMap(), metaPageAddr, metaIo, grpId, PageIdAllocator.INDEX_PARTITION,
-                                                    this.ctx.kernalContext().cache().context().pageStore().pages(grpId, PageIdAllocator.INDEX_PARTITION));
-                                    }
-                                    finally {
-                                        pageMem.writeUnlock(grpId, metaPageId, metaPage, null, true);
-                                    }
-                                }
-                                finally {
-                                    pageMem.releasePage(grpId, metaPageId, metaPage);
-                                }
+                                saveMeta(ctx);
 
                                 wasSaveToMeta = true;
                             }
 
-                            GridDhtPartitionMap partMap = grp.topology().localPartitionMap();
-
-                            if (partMap.containsKey(store.partId()) &&
-                                partMap.get(store.partId()) == GridDhtPartitionState.OWNING)
-                                addPartition(ctx.partitionStatMap(), partMetaPageAddr, io, grpId, store.partId(),
-                                    this.ctx.pageStore().pages(grpId, store.partId()));
+                            if (state == OWNING) {
+                                assert part != null;
+
+                                if(!addPartition(
+                                    part,
+                                    ctx.partitionStatMap(),
+                                    partMetaPageAddr,
+                                    io,
+                                    grpId,
+                                    store.partId(),
+                                    this.ctx.pageStore().pages(grpId, store.partId()),
+                                    store.fullSize()
+                                ))
+                                    U.warn(log,"Partition was concurrently evicted grpId=" +  grpId +
+                                            ", partitionId=" + part.id());
+                            }
+                            else if (state == MOVING || state == RENTING) {
+                                if (ctx.partitionStatMap().forceSkipIndexPartition(grpId)) {
+                                    if (log.isInfoEnabled())
+                                        log.info("Will not include SQL indexes to snapshot because there is " +
+                                            "a partition not in " + OWNING + " state [grp=" + grp.cacheOrGroupName() +
+                                            ", partId=" + store.partId() + ", state=" + state + ']');
+                                }
+                            }
 
                             changed = true;
                         }
@@ -386,47 +339,211 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
     }
 
     /**
-     * @param cacheSizes Cache sizes.
-     * @return Serialized cache sizes
+     * Loads cache sizes for all caches in shared group.
+     *
+     * @param pageMem page memory to perform operations on pages.
+     * @param grpId Cache group ID.
+     * @param cntrsPageId Counters page ID, if zero is provided that means no counters page exist.
+     * @return Cache sizes if store belongs to group containing multiple caches and sizes are available in memory. May
+     * return null if counter page does not exist.
+     * @throws IgniteCheckedException If page memory operation failed.
+     */
+    @Nullable private static Map<Integer, Long> readSharedGroupCacheSizes(PageSupport pageMem, int grpId,
+        long cntrsPageId) throws IgniteCheckedException {
+
+        if (cntrsPageId == 0L)
+            return null;
+
+        Map<Integer, Long> cacheSizes = new HashMap<>();
+
+        long nextId = cntrsPageId;
+
+        while (true){
+            final long curId = nextId;
+            final long curPage = pageMem.acquirePage(grpId, curId);
+
+            try {
+                final long curAddr = pageMem.readLock(grpId, curId, curPage);
+
+                assert curAddr != 0;
+
+                try {
+                    PagePartitionCountersIO cntrsIO = PageIO.getPageIO(curAddr);
+
+                    if (cntrsIO.readCacheSizes(curAddr, cacheSizes))
+                        break;
+
+                    nextId = cntrsIO.getNextCountersPageId(curAddr);
+
+                    assert nextId != 0;
+                }
+                finally {
+                    pageMem.readUnlock(grpId, curId, curPage);
+                }
+            }
+            finally {
+                pageMem.releasePage(grpId, curId, curPage);
+            }
+        }
+        return cacheSizes;
+    }
+
+    /**
+     * Saves cache sizes for all caches in shared group. Unconditionally marks pages as dirty.
+     *
+     * @param pageMem page memory to perform operations on pages.
+     * @param grpId Cache group ID.
+     * @param cntrsPageId Counters page ID, if zero is provided that means no counters page exist.
+     * @param partId Partition ID.
+     * @param sizes Cache sizes of all caches in group. Not null.
+     * @return new counter page Id. Same as {@code cntrsPageId} or new value if cache size pages were initialized.
+     * @throws IgniteCheckedException if page memory operation failed.
      */
-    private byte[] serializeCacheSizes(Map<Integer, Long> cacheSizes) {
-        // Item size = 4 bytes (cache ID) + 8 bytes (cache size) = 12 bytes
-        byte[] data = new byte[cacheSizes.size() * 12];
-        long off = GridUnsafe.BYTE_ARR_OFF;
+    private static long writeSharedGroupCacheSizes(PageMemory pageMem, int grpId,
+        long cntrsPageId, int partId, Map<Integer, Long> sizes) throws IgniteCheckedException {
+        byte[] data = PagePartitionCountersIO.VERSIONS.latest().serializeCacheSizes(sizes);
+
+        int items = data.length / PagePartitionCountersIO.ITEM_SIZE;
+        boolean init = cntrsPageId == 0;
+
+        if (init && !sizes.isEmpty())
+            cntrsPageId = pageMem.allocatePage(grpId, partId, PageIdAllocator.FLAG_DATA);
+
+        long nextId = cntrsPageId;
+        int written = 0;
+
+        while (written != items) {
+            final long curId = nextId;
+            final long curPage = pageMem.acquirePage(grpId, curId);
+
+            try {
+                final long curAddr = pageMem.writeLock(grpId, curId, curPage);
+
+                int pageSize = pageMem.pageSize();
+
+                assert curAddr != 0;
+
+                try {
+                    PagePartitionCountersIO partCntrIo;
+
+                    if (init) {
+                        partCntrIo = PagePartitionCountersIO.VERSIONS.latest();
+
+                        partCntrIo.initNewPage(curAddr, curId, pageSize);
+                    }
+                    else
+                        partCntrIo = PageIO.getPageIO(curAddr);
 
-        for (Map.Entry<Integer, Long> entry : cacheSizes.entrySet()) {
-            GridUnsafe.putInt(data, off, entry.getKey()); off += 4;
-            GridUnsafe.putLong(data, off, entry.getValue()); off += 8;
+                    written += partCntrIo.writeCacheSizes(pageSize, curAddr, data, written);
+
+                    nextId = partCntrIo.getNextCountersPageId(curAddr);
+
+                    if (written != items && (init = nextId == 0)) {
+                        //allocate new counters page
+                        nextId = pageMem.allocatePage(grpId, partId, PageIdAllocator.FLAG_DATA);
+                        partCntrIo.setNextCountersPageId(curAddr, nextId);
+                    }
+                }
+                finally {
+                    // Write full page
+                    pageMem.writeUnlock(grpId, curId, curPage, Boolean.TRUE, true);
+                }
+            }
+            finally {
+                pageMem.releasePage(grpId, curId, curPage);
+            }
         }
 
-        return data;
+        return cntrsPageId;
     }
 
     /**
+     * @param ctx Context.
+     */
+    private void saveMeta(Context ctx) throws IgniteCheckedException {
+        int grpId = grp.groupId();
+        PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
+        IgniteWriteAheadLogManager wal = this.ctx.wal();
+
+        long metaPageId = pageMem.metaPageId(grpId);
+        long metaPage = pageMem.acquirePage(grpId, metaPageId);
+
+        try {
+            long metaPageAddr = pageMem.writeLock(grpId, metaPageId, metaPage);
+
+            try {
+                PageMetaIO metaIo = PageMetaIO.getPageIO(metaPageAddr);
+
+                long nextSnapshotTag = metaIo.getNextSnapshotTag(metaPageAddr);
+
+                metaIo.setNextSnapshotTag(metaPageAddr, nextSnapshotTag + 1);
+
+                if (log != null && log.isDebugEnabled())
+                    log.debug("Save next snapshot before checkpoint start for grId = " + grpId
+                        + ", nextSnapshotTag = " + nextSnapshotTag);
+
+                if (PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, metaPageId,
+                    metaPage, wal, null))
+                    wal.log(new MetaPageUpdateNextSnapshotId(grpId, metaPageId,
+                        nextSnapshotTag + 1));
+
+                addPartition(
+                    null,
+                    ctx.partitionStatMap(),
+                    metaPageAddr,
+                    metaIo,
+                    grpId,
+                    PageIdAllocator.INDEX_PARTITION,
+                    this.ctx.pageStore().pages(grpId, PageIdAllocator.INDEX_PARTITION),
+                    -1);
+            }
+            finally {
+                pageMem.writeUnlock(grpId, metaPageId, metaPage, null, true);
+            }
+        }
+        finally {
+            pageMem.releasePage(grpId, metaPageId, metaPage);
+        }
+    }
+
+    /**
+     * @param part
      * @param map Map to add values to.
      * @param metaPageAddr Meta page address
      * @param io Page Meta IO
      * @param cacheId Cache ID.
-     * @param partId Partition ID. Or {@link PageIdAllocator#INDEX_PARTITION} for index partition
      * @param currAllocatedPageCnt total number of pages allocated for partition <code>[partition, cacheId]</code>
      */
-    private static void addPartition(
-        final PartitionAllocationMap map,
-        final long metaPageAddr,
-        final PageMetaIO io,
-        final int cacheId,
-        final int partId,
-        final int currAllocatedPageCnt
+    private static boolean addPartition(
+            GridDhtLocalPartition part,
+            final PartitionAllocationMap map,
+            final long metaPageAddr,
+            final PageMetaIO io,
+            final int cacheId,
+            final int partId,
+            final int currAllocatedPageCnt,
+            final int partSize
     ) {
-        if (currAllocatedPageCnt <= 1)
-            return;
+        if (part != null) {
+            boolean reserved = part.reserve();
+
+            if(!reserved)
+                return false;
+        }
+        else
+            assert partId == PageIdAllocator.INDEX_PARTITION : partId;
 
         assert PageIO.getPageId(metaPageAddr) != 0;
 
         int lastAllocatedPageCnt = io.getLastAllocatedPageCount(metaPageAddr);
+
+        int curPageCnt = lastAllocatedPageCnt == 0 && partSize == 0 ? 0 : currAllocatedPageCnt;
+
         map.put(
             new GroupPartitionId(cacheId, partId),
-            new PagesAllocationRange(lastAllocatedPageCnt, currAllocatedPageCnt));
+            new PagesAllocationRange(lastAllocatedPageCnt, curPageCnt));
+
+        return true;
     }
 
     /** {@inheritDoc} */
@@ -485,7 +602,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
         if (grp.sharedGroup())
             idxName = Integer.toString(cacheId) + "_" + idxName;
 
-        return metaStore.getOrAllocateForTree(idxName);
+        return indexStorage.getOrAllocateForTree(idxName);
     }
 
     /** {@inheritDoc} */
@@ -493,7 +610,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
         if (grp.sharedGroup())
             idxName = Integer.toString(cacheId) + "_" + idxName;
 
-        metaStore.dropRootPage(idxName);
+        indexStorage.dropRootPage(idxName);
     }
 
     /** {@inheritDoc} */
@@ -613,7 +730,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
         for (CacheDataStore store : partDataStores.values()) {
             assert store instanceof GridCacheDataStore;
 
-            FreeListImpl freeList = ((GridCacheDataStore)store).freeList;
+            CacheFreeListImpl freeList = ((GridCacheDataStore)store).freeList;
 
             if (freeList == null)
                 continue;
@@ -863,7 +980,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
         private String name;
 
         /** */
-        private volatile FreeListImpl freeList;
+        private volatile CacheFreeListImpl freeList;
 
         /** */
         private volatile CacheDataStore delegate;
@@ -913,7 +1030,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
 
                     RootPage reuseRoot = metas.reuseListRoot;
 
-                    freeList = new FreeListImpl(
+                    freeList = new CacheFreeListImpl(
                         grp.groupId(),
                         grp.cacheOrGroupName() + "-" + partId,
                         grp.dataRegion().memoryMetrics(),
@@ -964,43 +1081,8 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
 
                                 Map<Integer, Long> cacheSizes = null;
 
-                                if (grp.sharedGroup()) {
-                                    long cntrsPageId = io.getCountersPageId(pageAddr);
-
-                                    if (cntrsPageId != 0L) {
-                                        cacheSizes = new HashMap<>();
-
-                                        long nextId = cntrsPageId;
-
-                                        while (true){
-                                            final long curId = nextId;
-                                            final long curPage = pageMem.acquirePage(grpId, curId);
-
-                                            try {
-                                                final long curAddr = pageMem.readLock(grpId, curId, curPage);
-
-                                                assert curAddr != 0;
-
-                                                try {
-                                                    PagePartitionCountersIO cntrsIO = PageIO.getPageIO(curAddr);
-
-                                                    if (cntrsIO.readCacheSizes(curAddr, cacheSizes))
-                                                        break;
-
-                                                    nextId = cntrsIO.getNextCountersPageId(curAddr);
-
-                                                    assert nextId != 0;
-                                                }
-                                                finally {
-                                                    pageMem.readUnlock(grpId, curId, curPage);
-                                                }
-                                            }
-                                            finally {
-                                                pageMem.releasePage(grpId, curId, curPage);
-                                            }
-                                        }
-                                    }
-                                }
+                                if (grp.sharedGroup())
+                                    cacheSizes = readSharedGroupCacheSizes(pageMem, grpId, io.getCountersPageId(pageAddr));
 
                                 delegate0.init(io.getSize(pageAddr), io.getUpdateCounter(pageAddr), cacheSizes);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index 8771f6a..1260147 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -25,6 +25,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import javax.management.InstanceNotFoundException;
 import org.apache.ignite.DataRegionMetrics;
 import org.apache.ignite.DataStorageMetrics;
 import org.apache.ignite.IgniteCheckedException;
@@ -51,8 +52,9 @@ import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictio
 import org.apache.ignite.internal.processors.cache.persistence.evict.Random2LruPageEvictionTracker;
 import org.apache.ignite.internal.processors.cache.persistence.evict.RandomLruPageEvictionTracker;
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeListImpl;
 import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
-import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
 import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport;
 import org.apache.ignite.internal.util.typedef.F;
@@ -83,7 +85,10 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
     private static final long MAX_PAGE_MEMORY_INIT_SIZE_32_BIT = 2L * 1024 * 1024 * 1024;
 
     /** */
-    protected Map<String, DataRegion> dataRegionMap;
+    protected volatile Map<String, DataRegion> dataRegionMap;
+
+    /** */
+    private volatile boolean dataRegionsInitialized;
 
     /** */
     protected Map<String, DataRegionMetrics> memMetricsMap;
@@ -92,10 +97,10 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
     protected DataRegion dfltDataRegion;
 
     /** */
-    protected Map<String, FreeListImpl> freeListMap;
+    protected Map<String, CacheFreeListImpl> freeListMap;
 
     /** */
-    private FreeListImpl dfltFreeList;
+    private CacheFreeListImpl dfltFreeList;
 
     /** Page size from memory configuration, may be set only for fake(standalone) IgniteCacheDataBaseSharedManager */
     private int pageSize;
@@ -112,6 +117,8 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
         validateConfiguration(memCfg);
 
         pageSize = memCfg.getPageSize();
+
+        initDataRegions(memCfg);
     }
 
     /**
@@ -172,7 +179,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
 
             boolean persistenceEnabled = memPlcCfg.isPersistenceEnabled();
 
-            FreeListImpl freeList = new FreeListImpl(0,
+            CacheFreeListImpl freeList = new CacheFreeListImpl(0,
                     cctx.igniteInstanceName(),
                     memMetrics,
                     memPlc,
@@ -210,21 +217,25 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
      * @throws IgniteCheckedException If failed to initialize swap path.
      */
     protected void initDataRegions(DataStorageConfiguration memCfg) throws IgniteCheckedException {
+        if (dataRegionsInitialized)
+            return;
+
         DataRegionConfiguration[] dataRegionCfgs = memCfg.getDataRegionConfigurations();
 
         int dataRegions = dataRegionCfgs == null ? 0 : dataRegionCfgs.length;
 
-        dataRegionMap = U.newHashMap(2 + dataRegions);
-        memMetricsMap = U.newHashMap(2 + dataRegions);
+        dataRegionMap = U.newHashMap(3 + dataRegions);
+        memMetricsMap = U.newHashMap(3 + dataRegions);
 
         if (dataRegionCfgs != null) {
             for (DataRegionConfiguration dataRegionCfg : dataRegionCfgs)
-                addDataRegion(memCfg, dataRegionCfg);
+                addDataRegion(memCfg, dataRegionCfg, dataRegionCfg.isPersistenceEnabled());
         }
 
         addDataRegion(
             memCfg,
-            memCfg.getDefaultDataRegionConfiguration()
+            memCfg.getDefaultDataRegionConfiguration(),
+            memCfg.getDefaultDataRegionConfiguration().isPersistenceEnabled()
         );
 
         addDataRegion(
@@ -233,8 +244,11 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
                 memCfg.getSystemRegionInitialSize(),
                 memCfg.getSystemRegionMaxSize(),
                 CU.isPersistenceEnabled(memCfg)
-            )
+            ),
+            CU.isPersistenceEnabled(memCfg)
         );
+
+        dataRegionsInitialized = true;
     }
 
     /**
@@ -242,9 +256,10 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
      * @param dataRegionCfg Data region config.
      * @throws IgniteCheckedException If failed to initialize swap path.
      */
-    private void addDataRegion(
+    protected void addDataRegion(
         DataStorageConfiguration dataStorageCfg,
-        DataRegionConfiguration dataRegionCfg
+        DataRegionConfiguration dataRegionCfg,
+        boolean trackable
     ) throws IgniteCheckedException {
         String dataRegionName = dataRegionCfg.getName();
 
@@ -255,7 +270,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
 
         DataRegionMetricsImpl memMetrics = new DataRegionMetricsImpl(dataRegionCfg, fillFactorProvider(dataRegionCfg));
 
-        DataRegion memPlc = initMemory(dataStorageCfg, dataRegionCfg, memMetrics);
+        DataRegion memPlc = initMemory(dataStorageCfg, dataRegionCfg, memMetrics, trackable);
 
         dataRegionMap.put(dataRegionName, memPlc);
 
@@ -278,11 +293,11 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
         final String dataRegName = dataRegCfg.getName();
 
         return new IgniteOutClosure<Float>() {
-            private FreeListImpl freeList;
+            private CacheFreeListImpl freeList;
 
             @Override public Float apply() {
                 if (freeList == null) {
-                    FreeListImpl freeList0 = freeListMap.get(dataRegName);
+                    CacheFreeListImpl freeList0 = freeListMap.get(dataRegName);
 
                     if (freeList0 == null)
                         return (float) 0;
@@ -526,7 +541,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
      */
     public void dumpStatistics(IgniteLogger log) {
         if (freeListMap != null) {
-            for (FreeListImpl freeList : freeListMap.values())
+            for (CacheFreeListImpl freeList : freeListMap.values())
                 freeList.dumpStatistics(log);
         }
     }
@@ -641,6 +656,8 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
             dataRegionMap.clear();
 
             dataRegionMap = null;
+
+            dataRegionsInitialized = false;
         }
     }
 
@@ -661,6 +678,9 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
                     "DataRegionMetrics", name
                     ));
         }
+        catch (InstanceNotFoundException ignored) {
+            // We tried to unregister a non-existing MBean, not a big deal.
+        }
         catch (Throwable e) {
             U.error(log, "Failed to unregister MBean for memory metrics: " +
                 name, e);
@@ -798,7 +818,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
 
         int sysPageSize = pageMem.systemPageSize();
 
-        FreeListImpl freeListImpl = freeListMap.get(plcCfg.getName());
+        CacheFreeListImpl freeListImpl = freeListMap.get(plcCfg.getName());
 
         for (;;) {
             long allocatedPagesCnt = pageMem.loadedPages();
@@ -826,7 +846,8 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
     private DataRegion initMemory(
         DataStorageConfiguration memCfg,
         DataRegionConfiguration plcCfg,
-        DataRegionMetricsImpl memMetrics
+        DataRegionMetricsImpl memMetrics,
+        boolean trackable
     ) throws IgniteCheckedException {
         File allocPath = buildAllocPath(plcCfg);
 
@@ -836,7 +857,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
                 log,
                 allocPath);
 
-        PageMemory pageMem = createPageMemory(memProvider, memCfg, plcCfg, memMetrics);
+        PageMemory pageMem = createPageMemory(memProvider, memCfg, plcCfg, memMetrics, trackable);
 
         return new DataRegion(pageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg, pageMem));
     }
@@ -845,7 +866,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
      * @param plc data region Configuration.
      * @param pageMem Page memory.
      */
-    private PageEvictionTracker createPageEvictionTracker(DataRegionConfiguration plc, PageMemory pageMem) {
+    protected PageEvictionTracker createPageEvictionTracker(DataRegionConfiguration plc, PageMemory pageMem) {
         if (plc.getPageEvictionMode() == DataPageEvictionMode.DISABLED || plc.isPersistenceEnabled())
             return new NoOpPageEvictionTracker();
 
@@ -873,7 +894,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
      *
      * @throws IgniteCheckedException If resolving swap directory fails.
      */
-    @Nullable private File buildAllocPath(DataRegionConfiguration plc) throws IgniteCheckedException {
+    @Nullable protected File buildAllocPath(DataRegionConfiguration plc) throws IgniteCheckedException {
         String path = plc.getSwapPath();
 
         if (path == null)
@@ -901,7 +922,8 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
         DirectMemoryProvider memProvider,
         DataStorageConfiguration memCfg,
         DataRegionConfiguration memPlcCfg,
-        DataRegionMetricsImpl memMetrics
+        DataRegionMetricsImpl memMetrics,
+        boolean trackable
     ) {
         memMetrics.persistenceEnabled(false);
 
@@ -969,4 +991,11 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
     protected void setPageSize(int pageSize) {
         this.pageSize = pageSize;
     }
+
+    /**
+     * @return MetaStorage
+     */
+    public MetaStorage metaStorage() {
+        return null;
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorage.java
new file mode 100644
index 0000000..5141b04
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorage.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ * Meta store.
+ */
+public interface IndexStorage {
+    /**
+     * Get or allocate initial page for an index.
+     *
+     * @param idxName Index name.
+     * @return {@link RootPage} that keeps pageId, allocated flag that shows whether the page
+     *      was newly allocated, and rootId that is counter which increments each time new page allocated.
+     * @throws IgniteCheckedException If failed.
+     */
+    public RootPage getOrAllocateForTree(String idxName) throws IgniteCheckedException;
+
+    /**
+     * Deallocate index page and remove from tree.
+     *
+     * @param idxName Index name.
+     * @return Root ID or -1 if no page was removed.
+     * @throws IgniteCheckedException  If failed.
+     */
+    public RootPage dropRootPage(String idxName) throws IgniteCheckedException;
+
+    /**
+     * Destroy this meta store.
+     *
+     * @throws IgniteCheckedException  If failed.
+     */
+    public void destroy() throws IgniteCheckedException;
+}


[08/17] ignite git commit: IEP-4 Baseline topology for persistent caches (Phase 1) Contributed by: Dmitriy Govorukhin Dmitry Pavlov Eduard Shangareev Ily

Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
index 962c4ca..d30dddb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
@@ -21,7 +21,6 @@ import java.io.File;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.configuration.DataStorageConfiguration;
-import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.pagemem.wal.WALIterator;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
@@ -46,9 +45,9 @@ public class IgniteWalIteratorFactory {
     @Nullable private File binaryMetadataFileStoreDir;
 
     /**
-     * Folder specifying location of marshaller mapping file store. {@code null} means no specific folder is configured. <br>
-     * This folder should be specified for converting data entries into BinaryObjects.
-     * Providing {@code null} will disable unmarshall for non primitive objects, BinaryObjects will be provided
+     * Folder specifying location of marshaller mapping file store. {@code null} means no specific folder is configured.
+     * <br> This folder should be specified for converting data entries into BinaryObjects. Providing {@code null} will
+     * disable unmarshall for non primitive objects, BinaryObjects will be provided
      */
     @Nullable private File marshallerMappingFileStoreDir;
 
@@ -56,7 +55,10 @@ public class IgniteWalIteratorFactory {
     private boolean keepBinary;
 
     /** Factory to provide I/O interfaces for read/write operations with files */
-    private final FileIOFactory ioFactory;
+    private FileIOFactory ioFactory;
+
+    /** Wal records iterator buffer size */
+    private int bufSize = StandaloneWalRecordsIterator.DFLT_BUF_SIZE;
 
     /**
      * Creates WAL files iterator factory.
@@ -69,8 +71,8 @@ public class IgniteWalIteratorFactory {
      * subfolder and consistent ID subfolder. Note Consistent ID should be already masked and should not contain special
      * symbols. Providing {@code null} means no specific folder is configured. <br>
      * @param marshallerMappingFileStoreDir Folder specifying location of marshaller mapping file store. Should include
-     * "marshaller" subfolder. Providing {@code null} will disable unmarshall for non primitive objects,
-     * BinaryObjects will be provided
+     * "marshaller" subfolder. Providing {@code null} will disable unmarshall for non primitive objects, BinaryObjects
+     * will be provided
      * @param keepBinary {@code true} disables complex object unmarshall into source classes
      */
     public IgniteWalIteratorFactory(
@@ -148,8 +150,10 @@ public class IgniteWalIteratorFactory {
      * @return closable WAL records iterator, should be closed when non needed
      * @throws IgniteCheckedException if failed to read folder
      */
-    public WALIterator iteratorArchiveDirectory(@NotNull final File walDirWithConsistentId) throws IgniteCheckedException {
-        return new StandaloneWalRecordsIterator(walDirWithConsistentId, log, prepareSharedCtx(), ioFactory, keepBinary);
+    public WALIterator iteratorArchiveDirectory(
+        @NotNull final File walDirWithConsistentId) throws IgniteCheckedException {
+        return new StandaloneWalRecordsIterator(
+            walDirWithConsistentId, log, prepareSharedCtx(), ioFactory, keepBinary, bufSize);
     }
 
     /**
@@ -166,7 +170,7 @@ public class IgniteWalIteratorFactory {
      * @throws IgniteCheckedException if failed to read files
      */
     public WALIterator iteratorArchiveFiles(@NotNull final File... files) throws IgniteCheckedException {
-        return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), ioFactory, false, keepBinary, files);
+        return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), ioFactory, false, keepBinary, bufSize, files);
     }
 
     /**
@@ -183,7 +187,7 @@ public class IgniteWalIteratorFactory {
      * @throws IgniteCheckedException if failed to read files
      */
     public WALIterator iteratorWorkFiles(@NotNull final File... files) throws IgniteCheckedException {
-        return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), ioFactory, true, keepBinary, files);
+        return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), ioFactory, true, keepBinary, bufSize, files);
     }
 
     /**
@@ -202,4 +206,18 @@ public class IgniteWalIteratorFactory {
             null, null, null, null,
             null, null, null);
     }
+
+    /**
+     * @param ioFactory New factory to provide I/O interfaces for read/write operations with files
+     */
+    public void ioFactory(FileIOFactory ioFactory) {
+        this.ioFactory = ioFactory;
+    }
+
+    /**
+     * @param bufSize New wal records iterator buffer size
+     */
+    public void bufferSize(int bufSize) {
+        this.bufSize = bufSize;
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
index 80dfc5b..fa3f7f3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
@@ -77,6 +77,7 @@ import org.apache.ignite.internal.processors.security.GridSecurityProcessor;
 import org.apache.ignite.internal.processors.segmentation.GridSegmentationProcessor;
 import org.apache.ignite.internal.processors.service.GridServiceProcessor;
 import org.apache.ignite.internal.processors.session.GridTaskSessionProcessor;
+import org.apache.ignite.internal.processors.subscription.GridInternalSubscriptionProcessor;
 import org.apache.ignite.internal.processors.task.GridTaskProcessor;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
 import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions;
@@ -609,6 +610,11 @@ public class StandaloneGridKernalContext implements GridKernalContext {
     }
 
     /** {@inheritDoc} */
+    @Override public GridInternalSubscriptionProcessor internalSubscriptionProcessor() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
     @Override public PdsFoldersResolver pdsFolderResolver() {
         return new PdsFoldersResolver() {
             /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
index b36c2db..7740f28 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
@@ -60,12 +60,12 @@ import static org.apache.ignite.internal.processors.cache.persistence.wal.serial
  * Operates over one directory, does not provide start and end boundaries
  */
 class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
+    /** Record buffer size */
+    public static final int DFLT_BUF_SIZE = 2 * 1024 * 1024;
+
     /** */
     private static final long serialVersionUID = 0L;
 
-    /** Record buffer size */
-    private static final int BUF_SIZE = 2 * 1024 * 1024;
-
     /**
      * WAL files directory. Should already contain 'consistent ID' as subfolder.
      * <code>null</code> value means file-by-file iteration mode
@@ -80,38 +80,31 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
     @Nullable
     private List<FileWriteAheadLogManager.FileDescriptor> walFileDescriptors;
 
-    /**
-     * True if this iterator used for work dir, false for archive.
-     * In work dir mode exceptions come from record reading are ignored (file may be not completed).
-     * Index of file is taken from file itself, not from file name
-     */
-    private boolean workDir;
-
     /** Keep binary. This flag disables converting of non primitive types (BinaryObjects) */
     private boolean keepBinary;
 
     /**
      * Creates iterator in directory scan mode
-     *  @param walFilesDir Wal files directory. Should already contain node consistent ID as subfolder
+     * @param walFilesDir Wal files directory. Should already contain node consistent ID as subfolder
      * @param log Logger.
-     * @param sharedCtx Shared context. Cache processor is to be configured if Cache Object Key & Data Entry is
- * required.
+     * @param sharedCtx Shared context. Cache processor is to be configured if Cache Object Key & Data Entry is required.
      * @param ioFactory File I/O factory.
      * @param keepBinary  Keep binary. This flag disables converting of non primitive types
-     * (BinaryObjects will be used instead)
+     * @param bufSize Buffer size.
      */
     StandaloneWalRecordsIterator(
         @NotNull File walFilesDir,
         @NotNull IgniteLogger log,
         @NotNull GridCacheSharedContext sharedCtx,
         @NotNull FileIOFactory ioFactory,
-        boolean keepBinary
+        boolean keepBinary,
+        int bufSize
     ) throws IgniteCheckedException {
         super(log,
             sharedCtx,
             new RecordSerializerFactoryImpl(sharedCtx),
             ioFactory,
-            BUF_SIZE);
+            bufSize);
         this.keepBinary = keepBinary;
         init(walFilesDir, false, null);
         advance();
@@ -119,7 +112,7 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
 
     /**
      * Creates iterator in file-by-file iteration mode. Directory
-     *  @param log Logger.
+     * @param log Logger.
      * @param sharedCtx Shared context. Cache processor is to be configured if Cache Object Key & Data Entry is
      * required.
      * @param ioFactory File I/O factory.
@@ -134,14 +127,14 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
             @NotNull FileIOFactory ioFactory,
             boolean workDir,
             boolean keepBinary,
+            int bufSize,
             @NotNull File... walFiles) throws IgniteCheckedException {
         super(log,
             sharedCtx,
             new RecordSerializerFactoryImpl(sharedCtx),
             ioFactory,
-            BUF_SIZE);
+            bufSize);
 
-        this.workDir = workDir;
         this.keepBinary = keepBinary;
         init(null, workDir, walFiles);
         advance();
@@ -163,10 +156,8 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
             FileWriteAheadLogManager.FileDescriptor[] descs = loadFileDescriptors(walFilesDir);
             curWalSegmIdx = !F.isEmpty(descs) ? descs[0].getIdx() : 0;
             this.walFilesDir = walFilesDir;
-            this.workDir = false;
         }
         else {
-            this.workDir = workDir;
 
             if (workDir)
                 walFileDescriptors = scanIndexesFromFileHeaders(walFiles);
@@ -244,9 +235,14 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
         final FileWriteAheadLogManager.FileDescriptor fd;
 
         if (walFilesDir != null) {
-            fd = new FileWriteAheadLogManager.FileDescriptor(
-                new File(walFilesDir,
-                    FileWriteAheadLogManager.FileDescriptor.fileName(curWalSegmIdx)));
+            File segmentFile = new File(walFilesDir,
+                FileWriteAheadLogManager.FileDescriptor.fileName(curWalSegmIdx));
+
+            if (!segmentFile.exists())
+                segmentFile = new File(walFilesDir,
+                    FileWriteAheadLogManager.FileDescriptor.fileName(curWalSegmIdx) + ".zip");
+
+            fd = new FileWriteAheadLogManager.FileDescriptor(segmentFile);
         }
         else {
             if (walFileDescriptors.isEmpty())
@@ -285,6 +281,7 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
                 log.error("Failed to perform post processing for data record ", e);
             }
         }
+
         return super.postProcessRecord(rec);
     }
 
@@ -312,7 +309,13 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
 
             postProcessedEntries.add(postProcessedEntry);
         }
-        return new DataRecord(postProcessedEntries, dataRec.timestamp());
+
+        DataRecord res = new DataRecord(postProcessedEntries, dataRec.timestamp());
+
+        res.size(dataRec.size());
+        res.position(dataRec.position());
+
+        return res;
     }
 
     /**
@@ -368,18 +371,6 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
     }
 
     /** {@inheritDoc} */
-    @Override protected void handleRecordException(
-        @NotNull final Exception e,
-        @Nullable final FileWALPointer ptr) {
-        super.handleRecordException(e, ptr);
-        final RuntimeException ex = new RuntimeException("Record reading problem occurred at file pointer [" + ptr + "]:" + e.getMessage(), e);
-
-        ex.printStackTrace();
-        if (!workDir)
-            throw ex;
-    }
-
-    /** {@inheritDoc} */
     @Override protected void onClose() throws IgniteCheckedException {
         super.onClose();
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/BaselineTopologyRecordSerializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/BaselineTopologyRecordSerializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/BaselineTopologyRecordSerializer.java
new file mode 100644
index 0000000..94b51c5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/BaselineTopologyRecordSerializer.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.serializer;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Map;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.wal.record.BaselineTopologyRecord;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * {@link BaselineTopologyRecord} WAL serializer.
+ */
+public class BaselineTopologyRecordSerializer {
+    /** Cache shared context. */
+    private GridCacheSharedContext cctx;
+
+    /** Class loader to unmarshal consistent IDs. */
+    private ClassLoader clsLdr;
+
+    /**
+     * Create an instance of serializer.
+     *
+     * @param cctx Cache shared context.
+     */
+    public BaselineTopologyRecordSerializer(GridCacheSharedContext cctx) {
+        this.cctx = cctx;
+
+        clsLdr = U.resolveClassLoader(cctx.gridConfig());
+    }
+
+    /**
+     * Writes {@link BaselineTopologyRecord} to given buffer.
+     *
+     * @param rec {@link BaselineTopologyRecord} instance.
+     * @param buf Byte buffer.
+     * @throws IgniteCheckedException In case of fail.
+     */
+    public void write(BaselineTopologyRecord rec, ByteBuffer buf) throws IgniteCheckedException {
+        buf.putInt(rec.id());
+
+        Map<Short, Object> mapping = rec.mapping();
+
+        if (mapping != null && !mapping.isEmpty()) {
+            buf.putInt(mapping.size());
+
+            for (Map.Entry<Short, Object> e : mapping.entrySet()) {
+                buf.putShort(e.getKey());
+
+                writeConsistentId(e.getValue(), buf);
+            }
+        }
+        else
+            buf.putInt(0);
+    }
+
+    /**
+     * Reads {@link BaselineTopologyRecord} from given input.
+     *
+     * @param in Input
+     * @return BaselineTopologyRecord instance.
+     * @throws IOException In case of fail.
+     * @throws IgniteCheckedException In case of fail.
+     */
+    public BaselineTopologyRecord read(ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException {
+        int id = in.readInt();
+
+        int size = in.readInt();
+
+        Map<Short, Object> mapping = size > 0 ? U.<Short, Object>newHashMap(size) : null;
+
+        for (int i = 0; i < size; i++) {
+            short compactId = in.readShort();
+
+            Object consistentId = readConsistentId(in);
+
+            mapping.put(compactId, consistentId);
+        }
+
+        return new BaselineTopologyRecord(id, mapping);
+    }
+
+    /**
+     * Returns size of marshalled {@link BaselineTopologyRecord} in bytes.
+     *
+     * @param rec BaselineTopologyRecord instance.
+     * @return Size of BaselineTopologyRecord instance in bytes.
+     * @throws IgniteCheckedException In case of fail.
+     */
+    public int size(BaselineTopologyRecord rec) throws IgniteCheckedException {
+        int size = 0;
+
+        size += /* Baseline topology ID. */ 4;
+
+        size += /* Consistent ID mapping size. */ 4;
+
+        if (rec.mapping() != null) {
+            for (Object consistentId : rec.mapping().values()) {
+                size += /* Compact ID size */ 2;
+
+                size += marshalConsistentId(consistentId).length;
+            }
+        }
+
+        return size;
+    }
+
+    /**
+     * Write consistent id to given buffer.
+     *
+     * @param consistentId Consistent id.
+     * @param buf Byte buffer.
+     * @throws IgniteCheckedException In case of fail.
+     */
+    private void writeConsistentId(Object consistentId, ByteBuffer buf) throws IgniteCheckedException {
+        byte[] content = marshalConsistentId(consistentId);
+
+        buf.putInt(content.length);
+        buf.put(content);
+    }
+
+    /**
+     * Read consistent id from given input.
+     *
+     * @param in Input.
+     * @return Consistent id.
+     * @throws IOException In case of fail.
+     * @throws IgniteCheckedException In case of fail.
+     */
+    private Object readConsistentId(ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException {
+        int len = in.readInt();
+        in.ensure(len);
+
+        byte[] content = new byte[len];
+        in.readFully(content);
+
+        return cctx.marshaller().unmarshal(content, clsLdr);
+    }
+
+    /**
+     * Marshal consistent id to byte array.
+     *
+     * @param consistentId Consistent id.
+     * @return Marshalled byte array.
+     * @throws IgniteCheckedException In case of fail.
+     */
+    private byte[] marshalConsistentId(Object consistentId) throws IgniteCheckedException {
+        return cctx.marshaller().marshal(consistentId);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
index 12992a1..e8d116b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
@@ -35,6 +35,7 @@ import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
 import org.apache.ignite.internal.pagemem.wal.record.LazyDataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
+import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
 import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
 import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
@@ -112,11 +113,11 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
     private TxRecordSerializer txRecordSerializer;
 
     /**
-     * @param cctx Cctx.
+     * @param cctx Cache shared context.
      */
     public RecordDataV1Serializer(GridCacheSharedContext cctx) {
         this.cctx = cctx;
-        this.txRecordSerializer = new TxRecordSerializer(cctx);
+        this.txRecordSerializer = new TxRecordSerializer();
         this.co = cctx.kernalContext().cacheObjects();
         this.pageSize = cctx.database().pageSize();
     }
@@ -161,6 +162,12 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
 
                 return 4 + dataSize(dataRec);
 
+            case METASTORE_DATA_RECORD:
+                MetastoreDataRecord metastoreDataRec = (MetastoreDataRecord)record;
+
+                return  4 + metastoreDataRec.key().getBytes().length + 4 +
+                    (metastoreDataRec.value() != null ? metastoreDataRec.value().length : 0);
+
             case HEADER_RECORD:
                 return HEADER_RECORD_DATA_SIZE;
 
@@ -283,7 +290,7 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
                 return 0;
 
             case TX_RECORD:
-                return txRecordSerializer.sizeOfTxRecord((TxRecord)record);
+                return txRecordSerializer.size((TxRecord)record);
 
             default:
                 throw new UnsupportedOperationException("Type: " + record.type());
@@ -384,6 +391,31 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
 
                 break;
 
+            case METASTORE_DATA_RECORD:
+                int strLen = in.readInt();
+
+                byte[] strBytes = new byte[strLen];
+
+                in.readFully(strBytes);
+
+                String key = new String(strBytes);
+
+                int valLen = in.readInt();
+
+                assert valLen >= 0;
+
+                byte[] val;
+
+                if (valLen > 0) {
+                    val = new byte[valLen];
+
+                    in.readFully(val);
+                }
+                else
+                    val = null;
+
+                return new MetastoreDataRecord(key, val);
+
             case HEADER_RECORD:
                 long magic = in.readLong();
 
@@ -783,13 +815,16 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
 
             case PART_META_UPDATE_STATE:
                 cacheId = in.readInt();
+
                 partId = in.readInt();
 
                 state = in.readByte();
 
                 long updateCntr = in.readLong();
 
-                res = new PartitionMetaStateRecord(cacheId, partId, GridDhtPartitionState.fromOrdinal(state), updateCntr);
+                GridDhtPartitionState partState = GridDhtPartitionState.fromOrdinal(state);
+
+                res = new PartitionMetaStateRecord(cacheId, partId, partState, updateCntr);
 
                 break;
 
@@ -804,7 +839,7 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
                 throw new EOFException("END OF SEGMENT");
 
             case TX_RECORD:
-                res = txRecordSerializer.readTxRecord(in);
+                res = txRecordSerializer.read(in);
 
                 break;
 
@@ -906,6 +941,29 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
 
                 break;
 
+            case METASTORE_DATA_RECORD:
+                MetastoreDataRecord metastoreDataRecord = (MetastoreDataRecord)rec;
+
+                byte[] strBytes = metastoreDataRecord.key().getBytes();
+
+                buf.putInt(strBytes.length);
+                buf.put(strBytes);
+                if (metastoreDataRecord.value() != null) {
+                    buf.putInt(metastoreDataRecord.value().length);
+                    buf.put(metastoreDataRecord.value());
+                }
+                else
+                    buf.putInt(0);
+
+                break;
+
+            case HEADER_RECORD:
+                buf.putLong(HeaderRecord.REGULAR_MAGIC);
+
+                buf.putInt(((HeaderRecord)rec).version());
+
+                break;
+
             case DATA_PAGE_INSERT_RECORD:
                 DataPageInsertRecord diRec = (DataPageInsertRecord)rec;
 
@@ -1294,7 +1352,7 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
                 break;
 
             case TX_RECORD:
-                txRecordSerializer.writeTxRecord((TxRecord)rec, buf);
+                txRecordSerializer.write((TxRecord)rec, buf);
 
                 break;
 
@@ -1307,6 +1365,15 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
     }
 
     /**
+     * Return shared cache context.
+     *
+     * @return Shared cache context.
+     */
+    public GridCacheSharedContext cctx() {
+        return cctx;
+    }
+
+    /**
      * @param buf Buffer to write to.
      * @param entry Data entry.
      */
@@ -1558,5 +1625,4 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
 
         return size;
     }
-
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java
index 0e574f8..efba611 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java
@@ -17,16 +17,27 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.wal.serializer;
 
+import java.io.DataInput;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.UUID;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.wal.record.BaselineTopologyRecord;
+import org.apache.ignite.internal.pagemem.wal.record.CacheState;
+import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.ExchangeRecord;
 import org.apache.ignite.internal.pagemem.wal.record.SnapshotRecord;
+import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
 import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord;
 
 /**
@@ -39,6 +50,12 @@ public class RecordDataV2Serializer implements RecordDataSerializer {
     /** V1 data serializer delegate. */
     private final RecordDataV1Serializer delegateSerializer;
 
+    /** Serializer of {@link TxRecord} records. */
+    private final TxRecordSerializer txRecordSerializer;
+
+    /** Serializer of {@link BaselineTopologyRecord} records. */
+    private final BaselineTopologyRecordSerializer bltRecSerializer;
+
     /**
      * Create an instance of V2 data serializer.
      *
@@ -46,22 +63,45 @@ public class RecordDataV2Serializer implements RecordDataSerializer {
      */
     public RecordDataV2Serializer(RecordDataV1Serializer delegateSerializer) {
         this.delegateSerializer = delegateSerializer;
+        this.txRecordSerializer = new TxRecordSerializer();
+        this.bltRecSerializer = new BaselineTopologyRecordSerializer(delegateSerializer.cctx());
     }
 
     /** {@inheritDoc} */
-    @Override public int size(WALRecord record) throws IgniteCheckedException {
-        switch (record.type()) {
+    @Override public int size(WALRecord rec) throws IgniteCheckedException {
+        switch (rec.type()) {
             case HEADER_RECORD:
                 return HEADER_RECORD_DATA_SIZE;
 
+            case CHECKPOINT_RECORD:
+                CheckpointRecord cpRec = (CheckpointRecord)rec;
+
+                assert cpRec.checkpointMark() == null || cpRec.checkpointMark() instanceof FileWALPointer :
+                    "Invalid WAL record: " + cpRec;
+
+                int cacheStatesSize = cacheStatesSize(cpRec.cacheGroupStates());
+
+                FileWALPointer walPtr = (FileWALPointer)cpRec.checkpointMark();
+
+                return 18 + cacheStatesSize + (walPtr == null ? 0 : 16);
+
             case DATA_RECORD:
-                return delegateSerializer.size(record) + 8/*timestamp*/;
+                return delegateSerializer.size(rec) + 8/*timestamp*/;
 
             case SNAPSHOT:
                 return 8 + 1;
 
+            case EXCHANGE:
+               return 4 /*type*/ + 8 /*timestamp*/ + 2 /*constId*/;
+
+            case TX_RECORD:
+                return txRecordSerializer.size((TxRecord)rec);
+
+            case BASELINE_TOP_RECORD:
+                return bltRecSerializer.size((BaselineTopologyRecord)rec);
+
             default:
-                return delegateSerializer.size(record);
+                return delegateSerializer.size(rec);
         }
     }
 
@@ -71,6 +111,26 @@ public class RecordDataV2Serializer implements RecordDataSerializer {
         ByteBufferBackedDataInput in
     ) throws IOException, IgniteCheckedException {
         switch (type) {
+            case CHECKPOINT_RECORD:
+                long msb = in.readLong();
+                long lsb = in.readLong();
+                boolean hasPtr = in.readByte() != 0;
+                int idx0 = hasPtr ? in.readInt() : 0;
+                int off = hasPtr ? in.readInt() : 0;
+                int len = hasPtr ? in.readInt() : 0;
+
+                Map<Integer, CacheState> states = readPartitionStates(in);
+
+                boolean end = in.readByte() != 0;
+
+                FileWALPointer walPtr = hasPtr ? new FileWALPointer(idx0, off, len) : null;
+
+                CheckpointRecord cpRec = new CheckpointRecord(new UUID(msb, lsb), walPtr, end);
+
+                cpRec.cacheGroupStates(states);
+
+                return cpRec;
+
             case DATA_RECORD:
                 int entryCnt = in.readInt();
                 long timeStamp = in.readLong();
@@ -81,26 +141,65 @@ public class RecordDataV2Serializer implements RecordDataSerializer {
                     entries.add(delegateSerializer.readDataEntry(in));
 
                 return new DataRecord(entries, timeStamp);
-
             case SNAPSHOT:
                 long snpId = in.readLong();
                 byte full = in.readByte();
 
                 return new SnapshotRecord(snpId, full == 1);
 
+            case EXCHANGE:
+                int idx = in.readInt();
+                short constId = in.readShort();
+                long ts = in.readLong();
+
+                return new ExchangeRecord(constId, ExchangeRecord.Type.values()[idx], ts);
+
+            case TX_RECORD:
+                return txRecordSerializer.read(in);
+
+            case BASELINE_TOP_RECORD:
+                return bltRecSerializer.read(in);
+
             default:
                 return delegateSerializer.readRecord(type, in);
         }
+
     }
 
     /** {@inheritDoc} */
-    @Override public void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException {
-        if (record instanceof HeaderRecord)
+    @Override public void writeRecord(WALRecord rec, ByteBuffer buf) throws IgniteCheckedException {
+        if (rec instanceof HeaderRecord)
             throw new UnsupportedOperationException("Writing header records is forbidden since version 2 of serializer");
 
-        switch (record.type()) {
+        switch (rec.type()) {
+            case CHECKPOINT_RECORD:
+                CheckpointRecord cpRec = (CheckpointRecord)rec;
+
+                assert cpRec.checkpointMark() == null || cpRec.checkpointMark() instanceof FileWALPointer :
+                    "Invalid WAL record: " + cpRec;
+
+                FileWALPointer walPtr = (FileWALPointer)cpRec.checkpointMark();
+                UUID cpId = cpRec.checkpointId();
+
+                buf.putLong(cpId.getMostSignificantBits());
+                buf.putLong(cpId.getLeastSignificantBits());
+
+                buf.put(walPtr == null ? (byte)0 : 1);
+
+                if (walPtr != null) {
+                    buf.putLong(walPtr.index());
+                    buf.putInt(walPtr.fileOffset());
+                    buf.putInt(walPtr.length());
+                }
+
+                putCacheStates(buf, cpRec.cacheGroupStates());
+
+                buf.put(cpRec.end() ? (byte)1 : 0);
+
+                break;
+
             case DATA_RECORD:
-                DataRecord dataRec = (DataRecord)record;
+                DataRecord dataRec = (DataRecord)rec;
 
                 buf.putInt(dataRec.writeEntries().size());
                 buf.putLong(dataRec.timestamp());
@@ -111,15 +210,118 @@ public class RecordDataV2Serializer implements RecordDataSerializer {
                 break;
 
             case SNAPSHOT:
-                SnapshotRecord snpRec = (SnapshotRecord)record;
+                SnapshotRecord snpRec = (SnapshotRecord)rec;
 
                 buf.putLong(snpRec.getSnapshotId());
                 buf.put(snpRec.isFull() ? (byte)1 : 0);
 
                 break;
 
+            case EXCHANGE:
+                ExchangeRecord r = (ExchangeRecord)rec;
+
+                buf.putInt(r.getType().ordinal());
+                buf.putShort(r.getConstId());
+                buf.putLong(r.timestamp());
+
+                break;
+
+            case TX_RECORD:
+                txRecordSerializer.write((TxRecord)rec, buf);
+
+                break;
+
+            case BASELINE_TOP_RECORD:
+                bltRecSerializer.write((BaselineTopologyRecord)rec, buf);
+
+                break;
+
             default:
-                delegateSerializer.writeRecord(record, buf);
+                delegateSerializer.writeRecord(rec, buf);
+        }
+    }
+
+    /**
+     * @param buf Buffer to read from.
+     * @return Read map.
+     */
+    private Map<Integer, CacheState> readPartitionStates(DataInput buf) throws IOException {
+        int caches = buf.readShort() & 0xFFFF;
+
+        if (caches == 0)
+            return Collections.emptyMap();
+
+        Map<Integer, CacheState> states = new HashMap<>(caches, 1.0f);
+
+        for (int i = 0; i < caches; i++) {
+            int cacheId = buf.readInt();
+
+            int parts = buf.readShort() & 0xFFFF;
+
+            CacheState state = new CacheState(parts);
+
+            for (int p = 0; p < parts; p++) {
+                int partId = buf.readShort() & 0xFFFF;
+                long size = buf.readLong();
+                long partCntr = buf.readLong();
+                byte partState = buf.readByte();
+
+                state.addPartitionState(partId, size, partCntr, partState);
+            }
+
+            states.put(cacheId, state);
         }
+
+        return states;
+    }
+
+    /**
+     * @param states Cache states.
+     */
+    private static void putCacheStates(ByteBuffer buf, Map<Integer, CacheState> states) {
+        buf.putShort((short)states.size());
+
+        for (Map.Entry<Integer, CacheState> entry : states.entrySet()) {
+            buf.putInt(entry.getKey());
+
+            CacheState state = entry.getValue();
+
+            // Need 2 bytes for the number of partitions.
+            buf.putShort((short)state.size());
+
+            for (int i = 0; i < state.size(); i++) {
+                short partIdx = (short)state.partitionByIndex(i);
+
+                buf.putShort(partIdx);
+
+                buf.putLong(state.partitionSizeByIndex(i));
+                buf.putLong(state.partitionCounterByIndex(i));
+                buf.put(state.stateByIndex(i));
+            }
+        }
+    }
+
+    /**
+     * @param states Partition states.
+     * @return Size required to write partition states.
+     */
+    private int cacheStatesSize(Map<Integer, CacheState> states) {
+        // Need 4 bytes for the number of caches.
+        int size = 2;
+
+        for (Map.Entry<Integer, CacheState> entry : states.entrySet()) {
+            // Cache ID.
+            size += 4;
+
+            // Need 2 bytes for the number of partitions.
+            size += 2;
+
+            CacheState state = entry.getValue();
+
+            // 2 bytes partition ID, size and counter per partition, part state.
+            size += 19 * state.size();
+        }
+
+        return size;
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
index 11bd16b..5ae269b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
@@ -125,8 +125,10 @@ public class RecordV1Serializer implements RecordSerializer {
 
             final WALRecord rec = dataSerializer.readRecord(recType, in);
 
+            rec.position(ptr);
+
             if (recordFilter != null && !recordFilter.apply(rec.type(), ptr))
-                return new FilteredRecord();
+                return FilteredRecord.INSTANCE;
             else if (marshalledMode) {
                 ByteBuffer buf = heapTlb.get();
 
@@ -137,8 +139,6 @@ public class RecordV1Serializer implements RecordSerializer {
                 else
                     buf.clear();
 
-                rec.position(ptr);
-
                 writeRecord(rec, buf);
 
                 buf.flip();

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
index adfd701..feeb810 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
@@ -116,7 +116,7 @@ public class RecordV2Serializer implements RecordSerializer {
                 if (in.skipBytes(toSkip) < toSkip)
                     throw new EOFException("Reached end of file while reading record: " + ptr);
 
-                return new FilteredRecord();
+                return FilteredRecord.INSTANCE;
             }
             else if (marshalledMode) {
                 ByteBuffer buf = heapTlb.get();
@@ -144,8 +144,13 @@ public class RecordV2Serializer implements RecordSerializer {
 
                 return new MarshalledRecord(recType, ptr, buf);
             }
-            else
-                return dataSerializer.readRecord(recType, in);
+            else {
+                WALRecord rec = dataSerializer.readRecord(recType, in);
+
+                rec.position(ptr);
+
+                return rec;
+            }
 
         }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/TxRecordSerializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/TxRecordSerializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/TxRecordSerializer.java
index e8b324d..2f6d4c7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/TxRecordSerializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/TxRecordSerializer.java
@@ -21,11 +21,9 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.HashMap;
 import java.util.Map;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
@@ -36,60 +34,38 @@ import org.apache.ignite.transactions.TransactionState;
  * {@link TxRecord} WAL serializer.
  */
 public class TxRecordSerializer {
-    /** Cache shared context. */
-    private GridCacheSharedContext cctx;
-
-    /** Class loader to unmarshal consistent ids. */
-    private ClassLoader classLoader;
-
-    /**
-     * Create an instance of serializer.
-     *
-     * @param cctx Cache shared context.
-     */
-    public TxRecordSerializer(GridCacheSharedContext cctx) {
-        this.cctx = cctx;
-
-        classLoader = U.resolveClassLoader(cctx.gridConfig());
-    }
-
     /**
      * Writes {@link TxRecord} to given buffer.
      *
-     * @param record TxRecord.
+     * @param rec TxRecord.
      * @param buf Byte buffer.
      * @throws IgniteCheckedException In case of fail.
      */
-    public void writeTxRecord(TxRecord record, ByteBuffer buf) throws IgniteCheckedException {
-        buf.put((byte) record.state().ordinal());
-        RecordV1Serializer.putVersion(buf, record.nearXidVersion(), true);
-        RecordV1Serializer.putVersion(buf, record.writeVersion(), true);
+    public void write(TxRecord rec, ByteBuffer buf) throws IgniteCheckedException {
+        buf.put((byte)rec.state().ordinal());
+        RecordV1Serializer.putVersion(buf, rec.nearXidVersion(), true);
+        RecordV1Serializer.putVersion(buf, rec.writeVersion(), true);
+
+        Map<Short, Collection<Short>> participatingNodes = rec.participatingNodes();
 
-        if (record.participatingNodes() != null) {
-            buf.putInt(record.participatingNodes().keySet().size());
+        if (participatingNodes != null && !participatingNodes.isEmpty()) {
+            buf.putInt(participatingNodes.size());
 
-            for (Object primaryNode : record.participatingNodes().keySet()) {
-                writeConsistentId(primaryNode, buf);
+            for (Map.Entry<Short, Collection<Short>> e : participatingNodes.entrySet()) {
+                buf.putShort(e.getKey());
 
-                Collection<Object> backupNodes = record.participatingNodes().get(primaryNode);
+                Collection<Short> backupNodes = e.getValue();
 
                 buf.putInt(backupNodes.size());
 
-                for (Object backupNode : backupNodes)
-                    writeConsistentId(backupNode, buf);
+                for (short backupNode : backupNodes)
+                    buf.putShort(backupNode);
             }
         }
-        else {
-            // Put zero size of participating nodes.
-            buf.putInt(0);
-        }
-
-        buf.put((byte) (record.remote() ? 1 : 0));
+        else
+            buf.putInt(0); // Put zero size of participating nodes.
 
-        if (record.remote())
-            writeConsistentId(record.primaryNode(), buf);
-
-        buf.putLong(record.timestamp());
+        buf.putLong(rec.timestamp());
     }
 
     /**
@@ -100,7 +76,7 @@ public class TxRecordSerializer {
      * @throws IOException In case of fail.
      * @throws IgniteCheckedException In case of fail.
      */
-    public TxRecord readTxRecord(ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException {
+    public TxRecord read(ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException {
         byte txState = in.readByte();
         TransactionState state = TransactionState.fromOrdinal(txState);
 
@@ -108,17 +84,18 @@ public class TxRecordSerializer {
         GridCacheVersion writeVer = RecordV1Serializer.readVersion(in, true);
 
         int participatingNodesSize = in.readInt();
-        Map<Object, Collection<Object>> participatingNodes = new HashMap<>(2 * participatingNodesSize);
+
+        Map<Short, Collection<Short>> participatingNodes = U.newHashMap(participatingNodesSize);
 
         for (int i = 0; i < participatingNodesSize; i++) {
-            Object primaryNode = readConsistentId(in);
+            short primaryNode = in.readShort();
 
             int backupNodesSize = in.readInt();
 
-            Collection<Object> backupNodes = new ArrayList<>(backupNodesSize);
+            Collection<Short> backupNodes = new ArrayList<>(backupNodesSize);
 
             for (int j = 0; j < backupNodesSize; j++) {
-                Object backupNode = readConsistentId(in);
+                short backupNode = in.readShort();
 
                 backupNodes.add(backupNode);
             }
@@ -126,102 +103,41 @@ public class TxRecordSerializer {
             participatingNodes.put(primaryNode, backupNodes);
         }
 
-        boolean hasRemote = in.readByte() == 1;
-
-        Object primaryNode = null;
-
-        if (hasRemote)
-            primaryNode = readConsistentId(in);
-
-        long timestamp = in.readLong();
+        long ts = in.readLong();
 
-        return new TxRecord(state, nearXidVer, writeVer, participatingNodes, primaryNode, timestamp);
+        return new TxRecord(state, nearXidVer, writeVer, participatingNodes, ts);
     }
 
     /**
      * Returns size of marshalled {@link TxRecord} in bytes.
      *
-     * @param record TxRecord.
+     * @param rec TxRecord.
      * @return Size of TxRecord in bytes.
      * @throws IgniteCheckedException In case of fail.
      */
-    public int sizeOfTxRecord(TxRecord record) throws IgniteCheckedException {
+    public int size(TxRecord rec) throws IgniteCheckedException {
         int size = 0;
 
         size += /* transaction state. */ 1;
-        size += CacheVersionIO.size(record.nearXidVersion(), true);
-        size += CacheVersionIO.size(record.writeVersion(), true);
+        size += CacheVersionIO.size(rec.nearXidVersion(), true);
+        size += CacheVersionIO.size(rec.writeVersion(), true);
 
         size += /* primary nodes count. */ 4;
 
-        if (record.participatingNodes() != null) {
-            for (Object primaryNode : record.participatingNodes().keySet()) {
-                size += /* byte array length. */ 4;
-                size += marshalConsistentId(primaryNode).length;
+        Map<Short, Collection<Short>> participatingNodes = rec.participatingNodes();
 
-                Collection<Object> backupNodes = record.participatingNodes().get(primaryNode);
+        if (participatingNodes != null && !participatingNodes.isEmpty()) {
+            for (Collection<Short> backupNodes : participatingNodes.values()) {
+                size += /* Compact ID. */ 2;
 
                 size += /* size of backup nodes. */ 4;
 
-                for (Object backupNode : backupNodes) {
-                    size += /* byte array length. */ 4;
-                    size += marshalConsistentId(backupNode).length;
-                }
+                size += /* Compact ID. */ 2 * backupNodes.size();
             }
         }
 
-        size += /* Is primary node exist. */ 1;
-
-        if (record.remote()) {
-            size += /* byte array length. */ 4;
-            size += marshalConsistentId(record.primaryNode()).length;
-        }
-
         size += /* Timestamp */ 8;
 
         return size;
     }
-
-    /**
-     * Marshal consistent id to byte array.
-     *
-     * @param consistentId Consistent id.
-     * @return Marshalled byte array.
-     * @throws IgniteCheckedException In case of fail.
-     */
-    private byte[] marshalConsistentId(Object consistentId) throws IgniteCheckedException {
-        return cctx.marshaller().marshal(consistentId);
-    }
-
-    /**
-     * Read consistent id from given input.
-     *
-     * @param in Input.
-     * @return Consistent id.
-     * @throws IOException In case of fail.
-     * @throws IgniteCheckedException In case of fail.
-     */
-    private Object readConsistentId(ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException {
-        int len = in.readInt();
-        in.ensure(len);
-
-        byte[] content = new byte[len];
-        in.readFully(content);
-
-        return cctx.marshaller().unmarshal(content, classLoader);
-    }
-
-    /**
-     * Write consistent id to given buffer.
-     *
-     * @param consistentId Consistent id.
-     * @param buf Byte buffer.
-     * @throws IgniteCheckedException In case of fail.
-     */
-    private void writeConsistentId(Object consistentId, ByteBuffer buf) throws IgniteCheckedException {
-        byte[] content = marshalConsistentId(consistentId);
-
-        buf.putInt(content.length);
-        buf.put(content);
-    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index e862c0a..6c5922a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -230,6 +230,12 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
                     "Write-behind mode for the cache store also requires CacheConfiguration.setWriteThrough(true) " +
                     "property. Fix configuration for the cache: " + cfg.getName());
             }
+
+            if (cctx.group().persistenceEnabled() && (cfg.isWriteThrough() || cfg.isReadThrough()))
+                U.quietAndWarn(log,
+                    "Both Ignite native persistence and CacheStore are configured for cache '" + cfg.getName() + "'. " +
+                    "This configuration does not guarantee strict consistency between CacheStore and Ignite data " +
+                    "storage upon restarts. Consult documentation for more details.");
         }
 
         sesLsnrs = CU.startStoreSessionListeners(cctx.kernalContext(), cfg.getCacheStoreSessionListenerFactories());

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index 00c637e..0fe2dd4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -45,6 +45,7 @@ import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.managers.discovery.ConsistentIdMapper;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheInvokeEntry;
@@ -64,6 +65,7 @@ import org.apache.ignite.internal.processors.cache.version.GridCacheLazyPlainVer
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionedEntryEx;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
 import org.apache.ignite.internal.util.GridSetWrapper;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -1001,6 +1003,8 @@ public abstract class IgniteTxAdapter extends GridMetadataAwareAdapter implement
 
         boolean notify = false;
 
+        WALPointer ptr = null;
+
         synchronized (this) {
             prev = this.state;
 
@@ -1089,48 +1093,63 @@ public abstract class IgniteTxAdapter extends GridMetadataAwareAdapter implement
                     log.debug("Invalid transaction state transition [invalid=" + state + ", cur=" + this.state +
                         ", tx=" + this + ']');
             }
-        }
-
-        if (notify) {
-            GridFutureAdapter<IgniteInternalTx> fut = finFut;
 
-            if (fut != null)
-                fut.onDone(this);
-        }
+            if (valid) {
+                // Seal transactions maps.
+                if (state != ACTIVE && state != SUSPENDED)
+                    seal();
 
-        if (valid) {
-            // Seal transactions maps.
-            if (state != ACTIVE && state != SUSPENDED)
-                seal();
+                if (cctx.wal() != null && cctx.tm().logTxRecords()) {
+                    // Log tx state change to WAL.
+                    if (state == PREPARED || state == COMMITTED || state == ROLLED_BACK) {
+                        assert txNodes != null || state == ROLLED_BACK : "txNodes=" + txNodes + " state=" + state;
 
-            if (cctx.wal() != null && cctx.tm().logTxRecords()) {
-                // Log tx state change to WAL.
-                if (state == PREPARED || state == COMMITTED || state == ROLLED_BACK) {
-                    assert txNodes != null || state == ROLLED_BACK;
+                        BaselineTopology baselineTop = cctx.kernalContext().state().clusterState().baselineTopology();
 
-                    Map<Object, Collection<Object>> participatingNodes = consistentIdMapper
-                        .mapToConsistentIds(topVer, txNodes);
+                        Map<Short, Collection<Short>> participatingNodes = consistentIdMapper
+                            .mapToCompactIds(topVer, txNodes, baselineTop);
 
-                    TxRecord txRecord = new TxRecord(
+                        TxRecord txRecord = new TxRecord(
                             state,
                             nearXidVersion(),
                             writeVersion(),
-                            participatingNodes,
-                            remote() ? nodeId() : null
-                    );
+                            participatingNodes
+                        );
 
-                    try {
-                        cctx.wal().log(txRecord);
-                    }
-                    catch (IgniteCheckedException e) {
-                        U.error(log, "Failed to log TxRecord: " + txRecord, e);
+                        try {
+                            ptr = cctx.wal().log(txRecord);
+                        }
+                        catch (IgniteCheckedException e) {
+                            U.error(log, "Failed to log TxRecord: " + txRecord, e);
 
-                        throw new IgniteException("Failed to log TxRecord: " + txRecord, e);
+                            throw new IgniteException("Failed to log TxRecord: " + txRecord, e);
+                        }
                     }
                 }
             }
         }
 
+        if (valid) {
+            if (ptr != null && (state == COMMITTED || state == ROLLED_BACK))
+                try {
+                    cctx.wal().fsync(ptr);
+                }
+                catch (IgniteCheckedException e) {
+                    String msg = "Failed to fsync ptr: " + ptr;
+
+                    U.error(log, msg, e);
+
+                    throw new IgniteException(msg, e);
+                }
+        }
+
+        if (notify) {
+            GridFutureAdapter<IgniteInternalTx> fut = finFut;
+
+            if (fut != null)
+                fut.onDone(this);
+        }
+
         return valid;
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 7da4898..2a0b176 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -487,7 +487,7 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
 
         checkValid();
 
-        Collection<IgniteTxEntry> commitEntries = near() ? allEntries() : writeEntries();
+        Collection<IgniteTxEntry> commitEntries = (near() || cctx.snapshot().needTxReadLogging()) ? allEntries() : writeEntries();
 
         boolean empty = F.isEmpty(commitEntries);
 
@@ -648,19 +648,6 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
                                     if (dhtVer == null)
                                         dhtVer = explicitVer != null ? explicitVer : writeVersion();
 
-                                    if (cacheCtx.group().persistenceEnabled() && !writeEntries().isEmpty()
-                                        && op != NOOP && op != RELOAD && op != READ)
-                                        ptr = cctx.wal().log(new DataRecord(new DataEntry(
-                                            cacheCtx.cacheId(),
-                                            txEntry.key(),
-                                            val,
-                                            op,
-                                            nearXidVersion(),
-                                            writeVersion(),
-                                            0,
-                                            txEntry.key().partition(),
-                                            txEntry.updateCounter())));
-
                                     if (op == CREATE || op == UPDATE) {
                                         assert val != null : txEntry;
 
@@ -690,6 +677,9 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
                                         if (updRes.success())
                                             txEntry.updateCounter(updRes.updatePartitionCounter());
 
+                                        if (updRes.loggedPointer() != null)
+                                            ptr = updRes.loggedPointer();
+
                                         if (nearCached != null && updRes.success()) {
                                             nearCached.innerSet(
                                                 null,
@@ -738,6 +728,9 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
                                         if (updRes.success())
                                             txEntry.updateCounter(updRes.updatePartitionCounter());
 
+                                        if (updRes.loggedPointer() != null)
+                                            ptr = updRes.loggedPointer();
+
                                         if (nearCached != null && updRes.success()) {
                                             nearCached.innerRemove(
                                                 null,
@@ -766,6 +759,19 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
                                             nearCached.innerReload();
                                     }
                                     else if (op == READ) {
+                                        if (cacheCtx.group().persistenceEnabled() && cctx.snapshot().needTxReadLogging()) {
+                                            ptr = cctx.wal().log(new DataRecord(new DataEntry(
+                                                cacheCtx.cacheId(),
+                                                txEntry.key(),
+                                                val,
+                                                op,
+                                                nearXidVersion(),
+                                                writeVersion(),
+                                                0,
+                                                txEntry.key().partition(),
+                                                txEntry.updateCounter())));
+                                        }
+
                                         ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry);
 
                                         if (expiry != null) {
@@ -866,7 +872,7 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
                     }
                 }
 
-                if (ptr != null)
+                if (ptr != null && !cctx.tm().logTxRecords())
                     cctx.wal().fsync(ptr);
             }
             catch (StorageException e) {

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index a9353f4..932acf6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -1589,6 +1589,8 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
             GridCacheContext cacheCtx = txEntry1.context();
 
             while (true) {
+                cctx.database().checkpointReadLock();
+
                 try {
                     GridCacheEntryEx entry1 = txEntry1.cached();
 
@@ -1641,6 +1643,9 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
 
                     throw new IgniteCheckedException("Entry lock has been cancelled for transaction: " + tx);
                 }
+                finally {
+                    cctx.database().checkpointReadUnlock();
+                }
             }
         }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java
new file mode 100644
index 0000000..7651f6e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java
@@ -0,0 +1,235 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeJob;
+import org.apache.ignite.compute.ComputeJobAdapter;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.compute.ComputeJobResultPolicy;
+import org.apache.ignite.compute.ComputeTaskAdapter;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.CacheObjectUtils;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.lang.GridIterator;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+@GridInternal
+public class CollectConflictPartitionKeysTask extends ComputeTaskAdapter<PartitionKey,
+    Map<PartitionHashRecord, List<PartitionEntryHashRecord>>> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Injected logger. */
+    @LoggerResource
+    private IgniteLogger log;
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(
+        List<ClusterNode> subgrid, PartitionKey partKey) throws IgniteException {
+        Map<ComputeJob, ClusterNode> jobs = new HashMap<>();
+
+        for (ClusterNode node : subgrid)
+            jobs.put(new CollectConflictPartitionKeysTask.CollectPartitionEntryHashesJob(partKey), node);
+
+        return jobs;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Map<PartitionHashRecord, List<PartitionEntryHashRecord>> reduce(List<ComputeJobResult> results)
+        throws IgniteException {
+        Map<PartitionHashRecord, List<PartitionEntryHashRecord>> totalRes = new HashMap<>();
+
+        for (ComputeJobResult res : results) {
+            Map<PartitionHashRecord, List<PartitionEntryHashRecord>> nodeRes = res.getData();
+
+            totalRes.putAll(nodeRes);
+        }
+
+        Set<PartitionEntryHashRecord> commonEntries = null;
+
+        for (List<PartitionEntryHashRecord> nodeEntryHashRecords : totalRes.values()) {
+            HashSet<PartitionEntryHashRecord> set = new HashSet<>(nodeEntryHashRecords);
+
+            if (commonEntries == null)
+                commonEntries = set;
+            else
+                commonEntries.retainAll(set);
+        }
+
+        if (commonEntries == null)
+            return Collections.emptyMap();
+
+        Map<PartitionHashRecord, List<PartitionEntryHashRecord>> conflictsRes = new HashMap<>();
+
+        for (Map.Entry<PartitionHashRecord, List<PartitionEntryHashRecord>> e : totalRes.entrySet()) {
+            HashSet<PartitionEntryHashRecord> conflicts = new HashSet<>(e.getValue());
+
+            conflicts.removeAll(commonEntries);
+
+            if (!conflicts.isEmpty())
+                conflictsRes.put(e.getKey(), new ArrayList<>(conflicts));
+        }
+
+        return conflictsRes;
+    }
+
+    /** {@inheritDoc} */
+    @Override public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) {
+        ComputeJobResultPolicy superRes = super.result(res, rcvd);
+
+        // Deny failover.
+        if (superRes == ComputeJobResultPolicy.FAILOVER) {
+            superRes = ComputeJobResultPolicy.WAIT;
+
+            log.warning("CollectPartitionEntryHashesJob failed on node " +
+                "[consistentId=" + res.getNode().consistentId() + "]", res.getException());
+        }
+
+        return superRes;
+    }
+
+    /**
+     *
+     */
+    public static class CollectPartitionEntryHashesJob extends ComputeJobAdapter {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** Ignite instance. */
+        @IgniteInstanceResource
+        private IgniteEx ignite;
+
+        /** Injected logger. */
+        @LoggerResource
+        private IgniteLogger log;
+
+        /** Partition key. */
+        private PartitionKey partKey;
+
+        /**
+         * @param partKey Partition key.
+         */
+        private CollectPartitionEntryHashesJob(PartitionKey partKey) {
+            this.partKey = partKey;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Map<PartitionHashRecord, List<PartitionEntryHashRecord>> execute() throws IgniteException {
+            CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(partKey.groupId());
+
+            if (grpCtx == null)
+                return Collections.emptyMap();
+
+            partKey.groupName(grpCtx.cacheOrGroupName());
+
+            GridDhtLocalPartition part = grpCtx.topology().localPartition(partKey.partitionId());
+
+            if (part == null || !part.reserve())
+                return Collections.emptyMap();
+
+            int partHash = 0;
+            long partSize;
+            long updateCntrBefore;
+            List<PartitionEntryHashRecord> partEntryHashRecords;
+
+            try {
+                if (part.state() != GridDhtPartitionState.OWNING)
+                    return Collections.emptyMap();
+
+                updateCntrBefore = part.updateCounter();
+
+                partSize = part.dataStore().fullSize();
+
+                GridIterator<CacheDataRow> it = grpCtx.offheap().partitionIterator(part.id());
+
+                partEntryHashRecords = new ArrayList<>();
+
+                while (it.hasNextX()) {
+                    CacheDataRow row = it.nextX();
+
+                    partHash += row.key().hashCode();
+
+                    int valHash = Arrays.hashCode(row.value().valueBytes(grpCtx.cacheObjectContext()));
+                    partHash += valHash;
+
+                    int cacheId = row.cacheId() == 0 ? grpCtx.groupId() : row.cacheId();
+                    DynamicCacheDescriptor desc = ignite.context().cache().cacheDescriptor(cacheId);
+
+                    assert desc != null;
+
+                    Object o = CacheObjectUtils.unwrapBinaryIfNeeded(grpCtx.cacheObjectContext(), row.key(), true, true);
+
+                    partEntryHashRecords.add(new PartitionEntryHashRecord(
+                        cacheId, desc.cacheName(), row.key(), o.toString(),
+                        row.key().valueBytes(grpCtx.cacheObjectContext()), row.version(), valHash));
+                }
+
+                long updateCntrAfter = part.updateCounter();
+
+                if (updateCntrBefore != updateCntrAfter) {
+                    throw new IgniteException("Cluster is not idle: update counter of partition " + partKey.toString() +
+                        " changed during hash calculation [before=" + updateCntrBefore +
+                        ", after=" + updateCntrAfter + "]");
+                }
+            }
+            catch (IgniteCheckedException e) {
+                U.error(log, "Can't calculate partition hash " + partKey.toString(), e);
+
+                return Collections.emptyMap();
+            }
+            finally {
+                part.release();
+            }
+
+            Object consId = ignite.context().discovery().localNode().consistentId();
+
+            boolean isPrimary = part.primary(grpCtx.topology().readyTopologyVersion());
+
+            PartitionHashRecord partHashRec = new PartitionHashRecord(
+                partKey, isPrimary, consId, partHash, updateCntrBefore, partSize);
+
+            Map<PartitionHashRecord, List<PartitionEntryHashRecord>> res = new HashMap<>();
+
+            res.put(partHashRec, partEntryHashRecords);
+
+            return res;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionEntryHashRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionEntryHashRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionEntryHashRecord.java
new file mode 100644
index 0000000..6b0d345
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionEntryHashRecord.java
@@ -0,0 +1,237 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+public class PartitionEntryHashRecord implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Cache id. */
+    @GridToStringExclude
+    private final int cacheId;
+
+    /** Cache name. */
+    private final String cacheName;
+
+    /** Binary key. */
+    @GridToStringExclude
+    private final KeyCacheObject key;
+
+    /** Key string. */
+    @GridToStringExclude
+    private final String keyStr;
+
+    /** Key bytes. */
+    private final byte[] keyBytes;
+
+    /** Grid Cache Version. */
+    private final GridCacheVersion ver;
+
+    /** Value. */
+    @GridToStringExclude
+    private volatile CacheObject val;
+
+    /** Value bytes. */
+    private volatile byte[] valBytes;
+
+    /** Value string. */
+    @GridToStringExclude
+    private volatile String valStr;
+
+    /** Value hash. */
+    @GridToStringExclude
+    private final int valHash;
+
+    /**
+     * @param cacheId Cache id.
+     * @param cacheName Cache name.
+     * @param key Key.
+     * @param keyStr Key string.
+     * @param keyBytes Key bytes.
+     * @param ver Version.
+     * @param valHash Value hash.
+     * @param val Value.
+     * @param valStr Value string.
+     * @param valBytes Value bytes.
+     */
+    public PartitionEntryHashRecord(int cacheId, String cacheName, KeyCacheObject key, String keyStr,
+        byte[] keyBytes, GridCacheVersion ver, int valHash, CacheObject val, String valStr, byte[] valBytes) {
+        this.cacheId = cacheId;
+        this.cacheName = cacheName;
+        this.key = key;
+        this.keyStr = keyStr;
+        this.keyBytes = keyBytes;
+        this.ver = ver;
+        this.val = val;
+        this.valStr = valStr;
+        this.valHash = valHash;
+        this.valBytes = valBytes;
+    }
+
+    /**
+     * @param cacheId Cache id.
+     * @param cacheName Cache name.
+     * @param key Key.
+     * @param keyStr Key string.
+     * @param keyBytes Key bytes.
+     * @param ver Version.
+     * @param valHash Value hash.
+     */
+    public PartitionEntryHashRecord(int cacheId, String cacheName, KeyCacheObject key,
+        String keyStr, byte[] keyBytes, GridCacheVersion ver, int valHash) {
+        this(cacheId, cacheName, key, keyStr, keyBytes, ver, valHash, null, null, null);
+    }
+
+    /**
+     * @return Cache id.
+     */
+    public int cacheId() {
+        return cacheId;
+    }
+
+    /**
+     * @return Cache name.
+     */
+    public String cacheName() {
+        return cacheName;
+    }
+
+    /**
+     * @return Binary key.
+     */
+    public KeyCacheObject key() {
+        return key;
+    }
+
+    /**
+     * @return Key bytes.
+     */
+    public byte[] keyBytes() {
+        return keyBytes;
+    }
+
+    /**
+     * @return Grid Cache Version.
+     */
+    public GridCacheVersion version() {
+        return ver;
+    }
+
+    /**
+     * @return Value hash.
+     */
+    public int valueHash() {
+        return valHash;
+    }
+
+    /**
+     * @return Key string.
+     */
+    public String keyString() {
+        return keyStr;
+    }
+
+    /**
+     * @return Binary value.
+     */
+    public CacheObject value() {
+        return val;
+    }
+
+    /**
+     * @param val Value.
+     */
+    public void value(CacheObject val) {
+        this.val = val;
+    }
+
+    /**
+     * @return Value string.
+     */
+    public String valueString() {
+        return valStr;
+    }
+
+    /**
+     * @param valStr Value string.
+     */
+    public void valueString(String valStr) {
+        this.valStr = valStr;
+    }
+
+    /**
+     * @param valBytes Value bytes.
+     */
+    public void valueBytes(byte[] valBytes) {
+        this.valBytes = valBytes;
+    }
+
+    /**
+     * @return Value bytes.
+     */
+    public byte[] valueBytes() {
+        return valBytes;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        PartitionEntryHashRecord that = (PartitionEntryHashRecord)o;
+
+        if (cacheId != that.cacheId)
+            return false;
+        if (valHash != that.valHash)
+            return false;
+        if (!Arrays.equals(keyBytes, that.keyBytes))
+            return false;
+        return ver != null ? ver.equals(that.ver) : that.ver == null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int hashCode() {
+        int res = cacheId;
+        res = 31 * res + Arrays.hashCode(keyBytes);
+        res = 31 * res + (ver != null ? ver.hashCode() : 0);
+        res = 31 * res + valHash;
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(PartitionEntryHashRecord.class, this,
+            "key", keyStr,
+            "value", valStr,
+            "keyBytes", keyBytes != null ? U.byteArray2HexString(keyBytes) : null,
+            "valueBytes", valBytes != null ? U.byteArray2HexString(valBytes) : null);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecord.java
new file mode 100644
index 0000000..0f6e2e4
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecord.java
@@ -0,0 +1,130 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Record containing partition checksum, primary flag and consistent ID of owner.
+ */
+public class PartitionHashRecord implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Partition key. */
+    @GridToStringExclude
+    private final PartitionKey partKey;
+
+    /** Is primary flag. */
+    private final boolean isPrimary;
+
+    /** Consistent id. */
+    private final Object consistentId;
+
+    /** Partition hash. */
+    private final int partHash;
+
+    /** Update counter. */
+    private final long updateCntr;
+
+    /** Size. */
+    private final long size;
+
+    /**
+     * @param partKey Partition key.
+     * @param isPrimary Is primary.
+     * @param consistentId Consistent id.
+     * @param partHash Partition hash.
+     * @param updateCntr Update counter.
+     * @param size Size.
+     */
+    public PartitionHashRecord(PartitionKey partKey, boolean isPrimary,
+        Object consistentId, int partHash, long updateCntr, long size) {
+        this.partKey = partKey;
+        this.isPrimary = isPrimary;
+        this.consistentId = consistentId;
+        this.partHash = partHash;
+        this.updateCntr = updateCntr;
+        this.size = size;
+    }
+
+    /**
+     * @return Partition key.
+     */
+    public PartitionKey partitionKey() {
+        return partKey;
+    }
+
+    /**
+     * @return Is primary.
+     */
+    public boolean isPrimary() {
+        return isPrimary;
+    }
+
+    /**
+     * @return Consistent id.
+     */
+    public Object consistentId() {
+        return consistentId;
+    }
+
+    /**
+     * @return Partition hash.
+     */
+    public int partitionHash() {
+        return partHash;
+    }
+
+    /**
+     * @return Update counter.
+     */
+    public long updateCounter() {
+        return updateCntr;
+    }
+
+    /**
+     * @return Size.
+     */
+    public long size() {
+        return size;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(PartitionHashRecord.class, this, "consistentId", consistentId);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        PartitionHashRecord record = (PartitionHashRecord)o;
+
+        return consistentId.equals(record.consistentId);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int hashCode() {
+        return consistentId.hashCode();
+    }
+}


[06/17] ignite git commit: IEP-4 Baseline topology for persistent caches (Phase 1) Contributed by: Dmitriy Govorukhin Dmitry Pavlov Eduard Shangareev Ily

Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index 927fd90..54e0f56 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -17,22 +17,29 @@
 
 package org.apache.ignite.internal.processors.cluster;
 
+import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.BaselineNode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.cluster.ClusterGroupAdapter;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
@@ -46,6 +53,9 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.GridChangeGlobalStateMessageResponse;
 import org.apache.ignite.internal.processors.cache.StateChangeRequest;
 import org.apache.ignite.internal.processors.cache.StoredCacheData;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadOnlyMetastorage;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadWriteMetastorage;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -58,8 +68,12 @@ import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
 import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.spi.IgniteNodeValidationResult;
 import org.apache.ignite.spi.discovery.DiscoveryDataBag;
 import org.jetbrains.annotations.Nullable;
 
@@ -71,13 +85,31 @@ import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYS
 /**
  *
  */
-public class GridClusterStateProcessor extends GridProcessorAdapter {
+public class GridClusterStateProcessor extends GridProcessorAdapter implements IGridClusterStateProcessor, MetastorageLifecycleListener {
+    /** */
+    private static final String METASTORE_CURR_BLT_KEY = "metastoreBltKey";
+
+    /** */
+    private boolean inMemoryMode;
+
+    /**
+     * Compatibility mode flag. When node detects it runs in heterogeneous cluster (nodes of different versions),
+     * it should skip baseline topology operations.
+     */
+    private volatile boolean compatibilityMode;
+
     /** */
     private volatile DiscoveryDataClusterState globalState;
 
+    /** */
+    private final BaselineTopologyHistory bltHist = new BaselineTopologyHistory();
+
     /** Local action future. */
     private final AtomicReference<GridChangeGlobalStateFuture> stateChangeFut = new AtomicReference<>();
 
+    /** */
+    private final ConcurrentMap<UUID, GridFutureAdapter<Void>> transitionFuts = new ConcurrentHashMap<>();
+
     /** Future initialized if node joins when cluster state change is in progress. */
     private TransitionOnJoinWaitFuture joinFut;
 
@@ -89,6 +121,13 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
     @GridToStringExclude
     private GridCacheSharedContext<?, ?> sharedCtx;
 
+    /** Fully initialized metastorage. */
+    @GridToStringExclude
+    private ReadWriteMetastorage metastorage;
+
+    /** */
+    private final JdkMarshaller marsh = new JdkMarshaller();
+
     /** Listener. */
     private final GridLocalEventListener lsr = new GridLocalEventListener() {
         @Override public void onEvent(Event evt) {
@@ -115,12 +154,20 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
      */
     public GridClusterStateProcessor(GridKernalContext ctx) {
         super(ctx);
+
+        ctx.internalSubscriptionProcessor().registerMetastorageListener(this);
     }
 
     /**
-     * @return Cluster state to be used on public API.
+     * @return {@code True} if {@link IGridClusterStateProcessor} has detected that cluster is working
+     * in compatibility mode (nodes of different versions are joined to the cluster).
      */
-    public boolean publicApiActiveState() {
+    public boolean compatibilityMode() {
+        return compatibilityMode;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean publicApiActiveState(boolean waitForTransition) {
         if (ctx.isDaemon())
             return sendComputeCheckGlobalState();
 
@@ -133,19 +180,120 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
 
             if (transitionRes != null)
                 return transitionRes;
-            else
-                return false;
+            else {
+                if (waitForTransition) {
+                    GridFutureAdapter<Void> fut = transitionFuts.get(globalState.transitionRequestId());
+
+                    if (fut != null) {
+                        try {
+                            fut.get();
+                        }
+                        catch (IgniteCheckedException ex) {
+                            throw new IgniteException(ex);
+                        }
+                    }
+
+                    transitionRes = globalState.transitionResult();
+
+                    assert transitionRes != null;
+
+                    return transitionRes;
+                }
+                else
+                    return false;
+            }
         }
         else
             return globalState.active();
     }
 
     /** {@inheritDoc} */
+    @Override public void onReadyForRead(ReadOnlyMetastorage metastorage) throws IgniteCheckedException {
+        BaselineTopology blt = (BaselineTopology) metastorage.read(METASTORE_CURR_BLT_KEY);
+
+        if (blt != null) {
+            if (log.isInfoEnabled())
+                U.log(log, "Restoring history for BaselineTopology[id=" + blt.id() + "]");
+
+            bltHist.restoreHistory(metastorage, blt.id());
+        }
+
+        onStateRestored(blt);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onReadyForReadWrite(ReadWriteMetastorage metastorage) throws IgniteCheckedException {
+        this.metastorage = metastorage;
+
+        if (compatibilityMode) {
+            if (log.isInfoEnabled())
+                log.info("BaselineTopology won't be stored as this node is running in compatibility mode");
+
+            return;
+        }
+
+        writeBaselineTopology(globalState.baselineTopology(), null);
+
+        bltHist.flushHistoryItems(metastorage);
+    }
+
+    /**
+     * Resets branching history on current BaselineTopology.
+     *
+     * @throws IgniteCheckedException If write to metastore has failed.
+     */
+    public void resetBranchingHistory(long newBranchingHash) throws IgniteCheckedException {
+        globalState.baselineTopology().resetBranchingHistory(newBranchingHash);
+
+        writeBaselineTopology(globalState.baselineTopology(), null);
+
+        U.log(log,
+            String.format("Branching history of current BaselineTopology is reset to the value %d", newBranchingHash));
+    }
+
+    /**
+     * @param blt Blt.
+     */
+    private void writeBaselineTopology(BaselineTopology blt, BaselineTopologyHistoryItem prevBltHistItem) throws IgniteCheckedException {
+        assert metastorage != null;
+
+        sharedCtx.database().checkpointReadLock();
+
+        try {
+            if (blt != null) {
+                if (log.isInfoEnabled()) {
+                    U.log(log, "Writing BaselineTopology[id=" + blt.id() + "]");
+
+                    if (prevBltHistItem != null)
+                        U.log(log, "Writing BaselineTopologyHistoryItem[id=" + prevBltHistItem.id() + "]");
+                }
+
+                bltHist.writeHistoryItem(metastorage, prevBltHistItem);
+
+                metastorage.write(METASTORE_CURR_BLT_KEY, blt);
+            }
+            else {
+                if (log.isInfoEnabled())
+                    U.log(log, "Removing BaselineTopology and history");
+
+                metastorage.remove(METASTORE_CURR_BLT_KEY);
+
+                bltHist.removeHistory(metastorage);
+            }
+        }
+        finally {
+            sharedCtx.database().checkpointReadUnlock();
+        }
+    }
+
+    /** {@inheritDoc} */
     @Override public void start() throws IgniteCheckedException {
+        inMemoryMode = !CU.isPersistenceEnabled(ctx.config());
+
         // Start first node as inactive if persistence is enabled.
-        boolean activeOnStart = !CU.isPersistenceEnabled(ctx.config()) && ctx.config().isActiveOnStart();
+        boolean activeOnStart = inMemoryMode && ctx.config().isActiveOnStart();
 
-        globalState = DiscoveryDataClusterState.createState(activeOnStart);
+        globalState = DiscoveryDataClusterState.createState(activeOnStart, null);
 
         ctx.event().addLocalEventListener(lsr, EVT_NODE_LEFT, EVT_NODE_FAILED);
     }
@@ -160,25 +308,46 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
         super.onKernalStop(cancel);
     }
 
-    /**
-     * @param discoCache Discovery data cache.
-     * @return If transition is in progress returns future which is completed when transition finishes.
-     */
-    @Nullable public IgniteInternalFuture<Boolean> onLocalJoin(DiscoCache discoCache) {
-        if (globalState.transition()) {
-            joinFut = new TransitionOnJoinWaitFuture(globalState, discoCache);
+    /** {@inheritDoc} */
+    @Override @Nullable public IgniteInternalFuture<Boolean> onLocalJoin(DiscoCache discoCache) {
+        final DiscoveryDataClusterState state = globalState;
+
+        if (state.transition()) {
+            joinFut = new TransitionOnJoinWaitFuture(state, discoCache);
 
             return joinFut;
         }
+        else if (!ctx.clientNode()
+                && !ctx.isDaemon()
+                && ctx.config().isAutoActivationEnabled()
+                && !state.active()
+                && isBaselineSatisfied(state.baselineTopology(), discoCache.serverNodes()))
+                changeGlobalState0(true, state.baselineTopology(), false);
 
         return null;
     }
 
     /**
-     * @param node Failed node.
-     * @return Message if cluster state changed.
+     * Checks whether all conditions to meet BaselineTopology are satisfied.
      */
-    @Nullable public ChangeGlobalStateFinishMessage onNodeLeft(ClusterNode node) {
+    private boolean isBaselineSatisfied(BaselineTopology blt, List<ClusterNode> serverNodes) {
+        if (blt == null)
+            return false;
+
+        if (blt.consistentIds() == null)
+            return false;
+
+        if (//only node participating in BaselineTopology is allowed to send activation command...
+            blt.consistentIds().contains(ctx.discovery().localNode().consistentId())
+                //...and with this node BaselineTopology is reached
+                && blt.isSatisfied(serverNodes))
+            return true;
+
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override @Nullable public ChangeGlobalStateFinishMessage onNodeLeft(ClusterNode node) {
         if (globalState.transition()) {
             Set<UUID> nodes = globalState.transitionNodes();
 
@@ -187,7 +356,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
                     "Switching to inactive state.");
 
                 ChangeGlobalStateFinishMessage msg =
-                    new ChangeGlobalStateFinishMessage(globalState.transitionRequestId(), false);
+                    new ChangeGlobalStateFinishMessage(globalState.transitionRequestId(), false, false);
 
                 onStateFinishMessage(msg);
 
@@ -198,14 +367,16 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
         return null;
     }
 
-    /**
-     * @param msg Message.
-     */
-    public void onStateFinishMessage(ChangeGlobalStateFinishMessage msg) {
-        if (msg.requestId().equals(globalState.transitionRequestId())) {
+    /** {@inheritDoc} */
+    @Override public void onStateFinishMessage(ChangeGlobalStateFinishMessage msg) {
+        DiscoveryDataClusterState state = globalState;
+
+        if (msg.requestId().equals(state.transitionRequestId())) {
             log.info("Received state change finish message: " + msg.clusterActive());
 
-            globalState = DiscoveryDataClusterState.createState(msg.clusterActive());
+            globalState = globalState.finish(msg.success());
+
+            afterStateChangeFinished(msg.id(), msg.success());
 
             ctx.cache().onStateChangeFinish(msg);
 
@@ -213,22 +384,39 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
 
             if (joinFut != null)
                 joinFut.onDone(false);
+
+            GridFutureAdapter<Void> transitionFut = transitionFuts.remove(state.transitionRequestId());
+
+            if (transitionFut != null) {
+                state.setTransitionResult(msg.requestId(), msg.clusterActive());
+
+                transitionFut.onDone();
+            }
         }
         else
             U.warn(log, "Received state finish message with unexpected ID: " + msg);
     }
 
-    /**
-     * @param topVer Current topology version.
-     * @param msg Message.
-     * @param discoCache Current nodes.
-     * @return {@code True} if need start state change process.
-     */
-    public boolean onStateChangeMessage(AffinityTopologyVersion topVer,
+    /** */
+    protected void afterStateChangeFinished(IgniteUuid msgId, boolean success) {
+        // no-op
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean onStateChangeMessage(
+        AffinityTopologyVersion topVer,
         ChangeGlobalStateMessage msg,
-        DiscoCache discoCache) {
-        if (globalState.transition()) {
-            if (globalState.active() != msg.activate()) {
+        DiscoCache discoCache
+    ) {
+        DiscoveryDataClusterState state = globalState;
+
+        if (log.isInfoEnabled())
+            U.log(log, "Received " + prettyStr(msg.activate()) + " request with BaselineTopology" +
+                (msg.baselineTopology() == null ? ": null"
+                    : "[id=" + msg.baselineTopology().id() + "]"));
+
+        if (state.transition()) {
+            if (isApplicable(msg, state)) {
                 GridChangeGlobalStateFuture fut = changeStateFuture(msg);
 
                 if (fut != null)
@@ -237,27 +425,31 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
             else {
                 final GridChangeGlobalStateFuture stateFut = changeStateFuture(msg);
 
-                if (stateFut != null) {
-                    IgniteInternalFuture<?> exchFut = ctx.cache().context().exchange().affinityReadyFuture(
-                        globalState.transitionTopologyVersion());
+                GridFutureAdapter<Void> transitionFut = transitionFuts.get(state.transitionRequestId());
 
-                    if (exchFut == null)
-                        exchFut = new GridFinishedFuture<>();
+                if (stateFut != null && transitionFut != null) {
+                    transitionFut.listen(new IgniteInClosure<IgniteInternalFuture<Void>>() {
+                        @Override public void apply(IgniteInternalFuture<Void> fut) {
+                            try {
+                                fut.get();
+
+                                stateFut.onDone();
+                            }
+                            catch (Exception ex) {
+                                stateFut.onDone(ex);
+                            }
 
-                    exchFut.listen(new CI1<IgniteInternalFuture<?>>() {
-                        @Override public void apply(IgniteInternalFuture<?> exchFut) {
-                            stateFut.onDone();
                         }
                     });
                 }
             }
         }
         else {
-            if (globalState.active() != msg.activate()) {
+            if (isApplicable(msg, state)) {
                 ExchangeActions exchangeActions;
 
                 try {
-                    exchangeActions = ctx.cache().onStateChangeRequest(msg, topVer);
+                    exchangeActions = ctx.cache().onStateChangeRequest(msg, topVer, state);
                 }
                 catch (IgniteCheckedException e) {
                     GridChangeGlobalStateFuture fut = changeStateFuture(msg);
@@ -281,14 +473,25 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
                 if (log.isInfoEnabled())
                     log.info("Started state transition: " + msg.activate());
 
-                globalState = DiscoveryDataClusterState.createTransitionState(msg.activate(),
+                BaselineTopologyHistoryItem bltHistItem = BaselineTopologyHistoryItem.fromBaseline(
+                    globalState.baselineTopology());
+
+                transitionFuts.put(msg.requestId(), new GridFutureAdapter<Void>());
+
+                globalState = DiscoveryDataClusterState.createTransitionState(
+                    globalState,
+                    msg.activate(),
+                    msg.activate() ? msg.baselineTopology() : globalState.baselineTopology(),
                     msg.requestId(),
                     topVer,
                     nodeIds);
 
+                if (msg.forceChangeBaselineTopology())
+                    globalState.setTransitionResult(msg.requestId(), msg.activate());
+
                 AffinityTopologyVersion stateChangeTopVer = topVer.nextMinorVersion();
 
-                StateChangeRequest req = new StateChangeRequest(msg, stateChangeTopVer);
+                StateChangeRequest req = new StateChangeRequest(msg, bltHistItem, msg.activate() != state.active(), stateChangeTopVer);
 
                 exchangeActions.stateChangeRequest(req);
 
@@ -309,12 +512,34 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
     }
 
     /**
-     * @return Current cluster state, should be called only from discovery thread.
+     * @param msg State change message.
+     * @param state Current cluster state.
+     * @return {@code True} if state change from message can be applied to the current state.
+     */
+    protected boolean isApplicable(ChangeGlobalStateMessage msg, DiscoveryDataClusterState state) {
+        return !isEquivalent(msg, state);
+    }
+
+    /**
+     * @param msg State change message.
+     * @param state Current cluster state.
+     * @return {@code True} if states are equivalent.
      */
-    public DiscoveryDataClusterState clusterState() {
+    protected static boolean isEquivalent(ChangeGlobalStateMessage msg, DiscoveryDataClusterState state) {
+        return (msg.activate() == state.active() && BaselineTopology.equals(msg.baselineTopology(), state.baselineTopology()));
+    }
+
+    /** {@inheritDoc} */
+    @Override public DiscoveryDataClusterState clusterState() {
         return globalState;
     }
 
+    /** {@inheritDoc} */
+    @Override public DiscoveryDataClusterState pendingState(ChangeGlobalStateMessage stateMsg) {
+        return DiscoveryDataClusterState.createState(stateMsg.activate() || stateMsg.forceChangeBaselineTopology(),
+            stateMsg.baselineTopology());
+    }
+
     /**
      * @param msg State change message.
      * @return Local future for state change process.
@@ -346,15 +571,13 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
      * @param activate New state.
      * @return State change error.
      */
-    private IgniteCheckedException concurrentStateChangeError(boolean activate) {
+    protected IgniteCheckedException concurrentStateChangeError(boolean activate) {
         return new IgniteCheckedException("Failed to " + prettyStr(activate) +
             ", because another state change operation is currently in progress: " + prettyStr(!activate));
     }
 
-    /**
-     *
-     */
-    public void cacheProcessorStarted() {
+    /** {@inheritDoc} */
+    @Override public void cacheProcessorStarted() {
         cacheProc = ctx.cache();
         sharedCtx = cacheProc.context();
 
@@ -391,40 +614,192 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
     }
 
     /** {@inheritDoc} */
+    @Override public void collectJoiningNodeData(DiscoveryDataBag dataBag) {
+        try {
+            byte[] marshalledState = marsh.marshal(globalState);
+
+            dataBag.addJoiningNodeData(discoveryDataType().ordinal(), marshalledState);
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteException(e);
+        }
+    }
+
+    /** {@inheritDoc} */
     @Override public void collectGridNodeData(DiscoveryDataBag dataBag) {
-        if (!dataBag.commonDataCollectedFor(STATE_PROC.ordinal()))
-            dataBag.addGridCommonData(STATE_PROC.ordinal(), globalState);
+        if (!dataBag.commonDataCollectedFor(STATE_PROC.ordinal())) {
+            DiscoveryDataBag.JoiningNodeDiscoveryData joiningNodeData = dataBag.newJoinerDiscoveryData(STATE_PROC.ordinal());
+
+            BaselineTopologyHistory historyToSend = null;
+
+            if (joiningNodeData != null) {
+                if (!joiningNodeData.hasJoiningNodeData()) {
+                    //compatibility mode: old nodes don't send any data on join, so coordinator of new version
+                    //doesn't send BaselineTopology history, only its current globalState
+                    dataBag.addGridCommonData(STATE_PROC.ordinal(), globalState);
+
+                    return;
+                }
+
+                DiscoveryDataClusterState joiningNodeState = null;
+
+                try {
+                    if (joiningNodeData.joiningNodeData() != null)
+                        joiningNodeState = marsh.unmarshal(
+                            (byte[]) joiningNodeData.joiningNodeData(),
+                            U.resolveClassLoader(ctx.config()));
+                } catch (IgniteCheckedException e) {
+                    U.error(log, "Failed to unmarshal disco data from joining node: " + joiningNodeData.joiningNodeId());
+
+                    return;
+                }
+
+                if (!bltHist.isEmpty()) {
+                    if (joiningNodeState != null && joiningNodeState.baselineTopology() != null) {
+                        int lastId = joiningNodeState.baselineTopology().id();
+
+                        historyToSend = bltHist.tailFrom(lastId);
+                    }
+                    else
+                        historyToSend = bltHist;
+                }
+
+                dataBag.addGridCommonData(STATE_PROC.ordinal(), new BaselineStateAndHistoryData(globalState, historyToSend));
+            }
+        }
     }
 
     /** {@inheritDoc} */
     @Override public void onGridDataReceived(DiscoveryDataBag.GridDiscoveryData data) {
-        DiscoveryDataClusterState state = (DiscoveryDataClusterState)data.commonData();
+        if (data.commonData() instanceof DiscoveryDataClusterState) {
+            if (globalState != null && globalState.baselineTopology() != null)
+                //node with BaselineTopology is not allowed to join mixed cluster
+                // (where some nodes don't support BaselineTopology)
+                throw new IgniteException("Node with BaselineTopology cannot join" +
+                    " mixed cluster running in compatibility mode");
+
+            globalState = (DiscoveryDataClusterState) data.commonData();
+
+            compatibilityMode = true;
+
+            return;
+        }
+
+        BaselineStateAndHistoryData stateDiscoData = (BaselineStateAndHistoryData)data.commonData();
+
+        if (stateDiscoData != null) {
+            DiscoveryDataClusterState state = stateDiscoData.globalState;
+
+            if (state.transition())
+                transitionFuts.put(state.transitionRequestId(), new GridFutureAdapter<Void>());
 
-        if (state != null)
             globalState = state;
+
+            if (stateDiscoData.recentHistory != null) {
+                for (BaselineTopologyHistoryItem item : stateDiscoData.recentHistory.history())
+                    bltHist.bufferHistoryItemForStore(item);
+            }
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> changeGlobalState(
+        final boolean activate,
+        Collection<? extends BaselineNode> baselineNodes,
+        boolean forceChangeBaselineTopology
+    ) {
+        if (inMemoryMode)
+            return changeGlobalState0(activate, null, false);
+
+        BaselineTopology newBlt = compatibilityMode ? null :
+            calculateNewBaselineTopology(activate, baselineNodes, forceChangeBaselineTopology);
+
+        return changeGlobalState0(activate, newBlt, forceChangeBaselineTopology);
     }
 
     /**
-     * @param activate New cluster state.
-     * @return State change future.
+     *
      */
-    public IgniteInternalFuture<?> changeGlobalState(final boolean activate) {
+    private BaselineTopology calculateNewBaselineTopology(final boolean activate,
+        Collection<? extends BaselineNode> baselineNodes,
+        boolean forceChangeBaselineTopology) {
+        BaselineTopology newBlt;
+
+        BaselineTopology currentBlt = globalState.baselineTopology();
+
+        int newBltId = 0;
+
+        if (currentBlt != null)
+            newBltId = activate ? currentBlt.id() + 1 : currentBlt.id();
+
+        if (baselineNodes != null && !baselineNodes.isEmpty()) {
+            List<BaselineNode> baselineNodes0 = new ArrayList<>();
+
+            for (BaselineNode node : baselineNodes) {
+                if (node instanceof ClusterNode) {
+                    ClusterNode clusterNode = (ClusterNode) node;
+
+                    if (!clusterNode.isClient() && !clusterNode.isDaemon())
+                        baselineNodes0.add(node);
+                }
+                else
+                    baselineNodes0.add(node);
+            }
+
+            baselineNodes = baselineNodes0;
+        }
+
+        if (forceChangeBaselineTopology)
+            newBlt = BaselineTopology.build(baselineNodes, newBltId);
+        else if (activate) {
+            if (baselineNodes == null)
+                baselineNodes = baselineNodes();
+
+            if (currentBlt == null)
+                newBlt = BaselineTopology.build(baselineNodes, newBltId);
+            else {
+                newBlt = currentBlt;
+
+                newBlt.updateHistory(baselineNodes);
+            }
+        }
+        else
+            newBlt = null;
+
+        return newBlt;
+    }
+
+    /** */
+    private Collection<BaselineNode> baselineNodes() {
+        List<ClusterNode> clNodes = ctx.discovery().serverNodes(AffinityTopologyVersion.NONE);
+
+        ArrayList<BaselineNode> bltNodes = new ArrayList<>(clNodes.size());
+
+        for (ClusterNode clNode : clNodes)
+            bltNodes.add(clNode);
+
+        return bltNodes;
+    }
+
+    /** */
+    private IgniteInternalFuture<?> changeGlobalState0(final boolean activate,
+        BaselineTopology blt, boolean forceChangeBaselineTopology) {
         if (ctx.isDaemon() || ctx.clientNode()) {
             GridFutureAdapter<Void> fut = new GridFutureAdapter<>();
 
-            sendComputeChangeGlobalState(activate, fut);
+            sendComputeChangeGlobalState(activate, blt, forceChangeBaselineTopology, fut);
 
             return fut;
         }
 
         if (cacheProc.transactions().tx() != null || sharedCtx.lockedTopologyVersion(null) != null) {
             return new GridFinishedFuture<>(new IgniteCheckedException("Failed to " + prettyStr(activate) +
-                " cluster (must invoke the method outside of an active transaction or lock)."));
+                " cluster (must invoke the method outside of an active transaction)."));
         }
 
         DiscoveryDataClusterState curState = globalState;
 
-        if (!curState.transition() && curState.active() == activate)
+        if (!curState.transition() && curState.active() == activate && BaselineTopology.equals(curState.baselineTopology(), blt))
             return new GridFinishedFuture<>();
 
         GridChangeGlobalStateFuture startedFut = null;
@@ -473,9 +848,15 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
         ChangeGlobalStateMessage msg = new ChangeGlobalStateMessage(startedFut.requestId,
             ctx.localNodeId(),
             storedCfgs,
-            activate);
+            activate,
+            blt,
+            forceChangeBaselineTopology,
+            System.currentTimeMillis());
 
         try {
+            if (log.isInfoEnabled())
+                U.log(log, "Sending " + prettyStr(activate) + " request with BaselineTopology " + blt);
+
             ctx.discovery().sendCustomEvent(msg);
 
             if (ctx.isStopping())
@@ -488,14 +869,115 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
             startedFut.onDone(e);
         }
 
-        return startedFut;
+        return wrapStateChangeFuture(startedFut, msg);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode node, DiscoveryDataBag.JoiningNodeDiscoveryData discoData) {
+        if (node.isClient() || node.isDaemon())
+            return null;
+
+        if (discoData.joiningNodeData() == null) {
+            if (globalState.baselineTopology() != null) {
+                String msg = "Node not supporting BaselineTopology" +
+                    " is not allowed to join the cluster with BaselineTopology";
+
+                return new IgniteNodeValidationResult(node.id(), msg, msg);
+            }
+
+            return null;
+        }
+
+        DiscoveryDataClusterState joiningNodeState;
+
+        try {
+            joiningNodeState = marsh.unmarshal((byte[]) discoData.joiningNodeData(), Thread.currentThread().getContextClassLoader());
+        } catch (IgniteCheckedException e) {
+            String msg = "Error on unmarshalling discovery data " +
+                "from node " + node.consistentId() + ": " + e.getMessage() +
+                "; node is not allowed to join";
+
+            return new IgniteNodeValidationResult(node.id(), msg , msg);
+        }
+
+        if (joiningNodeState == null || joiningNodeState.baselineTopology() == null)
+            return null;
+
+        if (globalState == null || globalState.baselineTopology() == null) {
+            if (joiningNodeState != null && joiningNodeState.baselineTopology() != null) {
+                String msg = "Node with set up BaselineTopology is not allowed to join cluster without one: " + node.consistentId();
+
+                return new IgniteNodeValidationResult(node.id(), msg, msg);
+            }
+        }
+
+        BaselineTopology joiningNodeBlt = joiningNodeState.baselineTopology();
+        BaselineTopology clusterBlt = globalState.baselineTopology();
+
+        String recommendation = " Consider cleaning persistent storage of the node and adding it to the cluster again.";
+
+        if (joiningNodeBlt.id() > clusterBlt.id()) {
+            String msg = "BaselineTopology of joining node ("
+                + node.consistentId()
+                + ") is not compatible with BaselineTopology in the cluster."
+                + " Joining node BlT id (" + joiningNodeBlt.id()
+                + ") is greater than cluster BlT id (" + clusterBlt.id() + ")."
+                + " New BaselineTopology was set on joining node with set-baseline command."
+                + recommendation;
+
+            return new IgniteNodeValidationResult(node.id(), msg, msg);
+        }
+
+        if (joiningNodeBlt.id() == clusterBlt.id()) {
+            if (!clusterBlt.isCompatibleWith(joiningNodeBlt)) {
+                String msg = "BaselineTopology of joining node ("
+                    + node.consistentId()
+                    + " ) is not compatible with BaselineTopology in the cluster."
+                    + " Branching history of cluster BlT (" + clusterBlt.branchingHistory()
+                    + ") doesn't contain branching point hash of joining node BlT ("
+                    + joiningNodeBlt.branchingPointHash()
+                    + ")." + recommendation;
+
+                return new IgniteNodeValidationResult(node.id(), msg, msg);
+            }
+        }
+        else if (joiningNodeBlt.id() < clusterBlt.id()) {
+            if (!bltHist.isCompatibleWith(joiningNodeBlt)) {
+                String msg = "BaselineTopology of joining node ("
+                    + node.consistentId()
+                    + ") is not compatible with BaselineTopology in the cluster."
+                    + " BlT id of joining node (" + joiningNodeBlt.id()
+                    + ") less than BlT id of cluster (" + clusterBlt.id()
+                    + ") but cluster's BaselineHistory doesn't contain branching point hash of joining node BlT ("
+                    + joiningNodeBlt.branchingPointHash()
+                    + ")." + recommendation;
+
+                return new IgniteNodeValidationResult(node.id(), msg, msg);
+            }
+        }
+
+        return null;
+    }
+
+    /**
+     * @param fut Original state change future.
+     * @param msg State change message.
+     * @return Wrapped state change future.
+     */
+    protected IgniteInternalFuture<?> wrapStateChangeFuture(IgniteInternalFuture fut, ChangeGlobalStateMessage msg) {
+        return fut;
     }
 
     /**
      * @param activate New cluster state.
      * @param resFut State change future.
      */
-    private void sendComputeChangeGlobalState(boolean activate, final GridFutureAdapter<Void> resFut) {
+    private void sendComputeChangeGlobalState(
+        boolean activate,
+        BaselineTopology blt,
+        boolean forceBlt,
+        final GridFutureAdapter<Void> resFut
+    ) {
         AffinityTopologyVersion topVer = ctx.discovery().topologyVersionEx();
 
         if (log.isInfoEnabled()) {
@@ -507,7 +989,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
 
         IgniteCompute comp = ((ClusterGroupAdapter)ctx.cluster().get().forServers()).compute();
 
-        IgniteFuture<Void> fut = comp.runAsync(new ClientChangeGlobalStateComputeRequest(activate));
+        IgniteFuture<Void> fut = comp.runAsync(new ClientChangeGlobalStateComputeRequest(activate, blt, forceBlt));
 
         fut.listen(new CI1<IgniteFuture>() {
             @Override public void apply(IgniteFuture fut) {
@@ -549,32 +1031,31 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
         });
     }
 
-    /**
-     * @param errs Errors.
-     * @param req State change request.
-     */
-    public void onStateChangeError(Map<UUID, Exception> errs, StateChangeRequest req) {
+    /** {@inheritDoc} */
+    @Override public void onStateChangeError(Map<UUID, Exception> errs, StateChangeRequest req) {
         assert !F.isEmpty(errs);
 
         // Revert caches start if activation request fail.
-        if (req.activate()) {
-            try {
-                cacheProc.onKernalStopCaches(true);
+        if (req.activeChanged()) {
+            if (req.activate()) {
+                try {
+                    cacheProc.onKernalStopCaches(true);
 
-                cacheProc.stopCaches(true);
+                    cacheProc.stopCaches(true);
 
-                sharedCtx.affinity().removeAllCacheInfo();
+                    sharedCtx.affinity().removeAllCacheInfo();
 
-                if (!ctx.clientNode())
-                    sharedCtx.deactivate();
+                    if (!ctx.clientNode())
+                        sharedCtx.deactivate();
+                }
+                catch (Exception e) {
+                    U.error(log, "Failed to revert activation request changes", e);
+                }
             }
-            catch (Exception e) {
-                U.error(log, "Failed to revert activation request changes", e);
+            else {
+                //todo https://issues.apache.org/jira/browse/IGNITE-5480
             }
         }
-        else {
-            //todo https://issues.apache.org/jira/browse/IGNITE-5480
-        }
 
         GridChangeGlobalStateFuture fut = changeStateFuture(req.initiatorNodeId(), req.requestId());
 
@@ -596,6 +1077,8 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
      * @param req State change request.
      */
     private void onFinalActivate(final StateChangeRequest req) {
+        ctx.dataStructures().onBeforeActivate();
+
         ctx.closure().runLocalSafe(new Runnable() {
             @Override public void run() {
                 boolean client = ctx.clientNode();
@@ -618,37 +1101,44 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
                             + ctx.localNodeId() + ", client=" + client + ", topVer=" + req.topologyVersion() + "]");
                 }
                 catch (Exception ex) {
-                    e = new IgniteCheckedException("Failed to perform final activation steps", ex);
-
-                    U.error(log, "Failed to perform final activation steps [nodeId=" + ctx.localNodeId() +
-                        ", client=" + client + ", topVer=" + req.topologyVersion() + "]", ex);
-                }
-                finally {
-                    globalState.setTransitionResult(req.requestId(), true);
-
-                    sendChangeGlobalStateResponse(req.requestId(), req.initiatorNodeId(), e);
+                    throw new IgniteException(ex);
                 }
             }
         });
     }
 
-    /**
-     * @param req State change request.
-     */
-    private void onFinalDeActivate(final StateChangeRequest req) {
-        globalState.setTransitionResult(req.requestId(), false);
+    /** {@inheritDoc} */
+    @Override public void onStateChangeExchangeDone(StateChangeRequest req) {
+        try {
+            if (req.activeChanged()) {
+                if (req.activate())
+                    onFinalActivate(req);
+
+                globalState.setTransitionResult(req.requestId(), req.activate());
+            }
+
+            sendChangeGlobalStateResponse(req.requestId(), req.initiatorNodeId(), null);
+        }
+        catch (Exception ex) {
+            Exception e = new IgniteCheckedException("Failed to perform final activation steps", ex);
 
-        sendChangeGlobalStateResponse(req.requestId(), req.initiatorNodeId(), null);
+            U.error(log, "Failed to perform final activation steps [nodeId=" + ctx.localNodeId() +
+                ", client=" + ctx.clientNode() + ", topVer=" + req.topologyVersion() + "]", ex);
+
+            sendChangeGlobalStateResponse(req.requestId(), req.initiatorNodeId(), e);
+        }
     }
 
-    /**
-     * @param req State change request.
-     */
-    public void onStateChangeExchangeDone(StateChangeRequest req) {
-        if (req.activate())
-            onFinalActivate(req);
-        else
-            onFinalDeActivate(req);
+    /** {@inheritDoc} */
+    @Override public void onBaselineTopologyChanged(BaselineTopology blt, BaselineTopologyHistoryItem prevBltHistItem) throws IgniteCheckedException {
+        if (compatibilityMode) {
+            if (log.isDebugEnabled())
+                log.info("BaselineTopology won't be stored as this node is running in compatibility mode");
+
+            return;
+        }
+
+        writeBaselineTopology(blt, prevBltHistItem);
     }
 
     /**
@@ -718,6 +1208,27 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
         }
     }
 
+    /** */
+    private void onStateRestored(BaselineTopology blt) {
+        DiscoveryDataClusterState state = globalState;
+
+        if (!state.active() && !state.transition() && state.baselineTopology() == null) {
+            DiscoveryDataClusterState newState = DiscoveryDataClusterState.createState(false, blt);
+
+            globalState = newState;
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onExchangeFinishedOnCoordinator(IgniteInternalFuture exchangeFuture, boolean hasMovingPartitions) {
+        // no-op
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean evictionsAllowed() {
+        return true;
+    }
+
     /**
      * @param activate Activate.
      * @return Activate flag string.
@@ -734,7 +1245,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
     /**
      *
      */
-    private static class GridChangeGlobalStateFuture extends GridFutureAdapter<Void> {
+    private class GridChangeGlobalStateFuture extends GridFutureAdapter<Void> {
         /** Request id. */
         @GridToStringInclude
         private final UUID requestId;
@@ -868,7 +1379,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
         /** {@inheritDoc} */
         @Override public boolean onDone(@Nullable Void res, @Nullable Throwable err) {
             if (super.onDone(res, err)) {
-                ctx.state().stateChangeFut.compareAndSet(this, null);
+                stateChangeFut.compareAndSet(this, null);
 
                 return true;
             }
@@ -892,20 +1403,37 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
         /** */
         private final boolean activate;
 
+        /** */
+        private final BaselineTopology baselineTopology;
+
+        /** */
+        private final boolean forceChangeBaselineTopology;
+
         /** Ignite. */
         @IgniteInstanceResource
-        private Ignite ig;
+        private IgniteEx ig;
 
         /**
          * @param activate New cluster state.
          */
-        private ClientChangeGlobalStateComputeRequest(boolean activate) {
+        private ClientChangeGlobalStateComputeRequest(boolean activate, BaselineTopology blt, boolean forceBlt) {
             this.activate = activate;
+            this.baselineTopology = blt;
+            this.forceChangeBaselineTopology = forceBlt;
         }
 
         /** {@inheritDoc} */
         @Override public void run() {
-            ig.active(activate);
+            try {
+                ig.context().state().changeGlobalState(
+                    activate,
+                    baselineTopology != null ? baselineTopology.currentBaseline() : null,
+                    forceChangeBaselineTopology
+                ).get();
+            }
+            catch (IgniteCheckedException ex) {
+                throw new IgniteException(ex);
+            }
         }
     }
 
@@ -961,4 +1489,22 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
             return false;
         }
     }
+
+    /** */
+    private static class BaselineStateAndHistoryData implements Serializable {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private final DiscoveryDataClusterState globalState;
+
+        /** */
+        private final BaselineTopologyHistory recentHistory;
+
+        /** */
+        BaselineStateAndHistoryData(DiscoveryDataClusterState globalState, BaselineTopologyHistory recentHistory) {
+            this.globalState = globalState;
+            this.recentHistory = recentHistory;
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IGridClusterStateProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IGridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IGridClusterStateProcessor.java
new file mode 100644
index 0000000..bc72a51
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IGridClusterStateProcessor.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.processors.cluster;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.processors.GridProcessor;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.StateChangeRequest;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public interface IGridClusterStateProcessor extends GridProcessor {
+    /**
+     * @return Cluster state to be used on public API.
+     */
+    boolean publicApiActiveState(boolean waitForTransition);
+
+    /**
+     * @param discoCache Discovery data cache.
+     * @return If transition is in progress returns future which is completed when transition finishes.
+     */
+    @Nullable IgniteInternalFuture<Boolean> onLocalJoin(DiscoCache discoCache);
+
+    /**
+     * @param node Failed node.
+     * @return Message if cluster state changed.
+     */
+    @Nullable ChangeGlobalStateFinishMessage onNodeLeft(ClusterNode node);
+
+    /**
+     * @param msg Message.
+     */
+    void onStateFinishMessage(ChangeGlobalStateFinishMessage msg);
+
+    /**
+     * @param topVer Current topology version.
+     * @param msg Message.
+     * @param discoCache Current nodes.
+     * @return {@code True} if need start state change process.
+     */
+    boolean onStateChangeMessage(AffinityTopologyVersion topVer,
+        ChangeGlobalStateMessage msg,
+        DiscoCache discoCache);
+
+    /**
+     * @return Current cluster state, should be called only from discovery thread.
+     */
+    DiscoveryDataClusterState clusterState();
+
+    /**
+     * @return Pending cluster state which will be used when state transition is finished.
+     */
+    DiscoveryDataClusterState pendingState(ChangeGlobalStateMessage stateMsg);
+
+    /**
+     *
+     */
+    void cacheProcessorStarted();
+
+    /**
+     * @param activate New cluster state.
+     * @return State change future.
+     */
+    IgniteInternalFuture<?> changeGlobalState(
+        boolean activate,
+        Collection<? extends BaselineNode> baselineNodes,
+        boolean forceChangeBaselineTopology
+    );
+
+    /**
+     * @param errs Errors.
+     * @param req State change request.
+     */
+    void onStateChangeError(Map<UUID, Exception> errs, StateChangeRequest req);
+
+    /**
+     * @param req State change request.
+     */
+    void onStateChangeExchangeDone(StateChangeRequest req);
+
+    /**
+     * @param blt New baseline topology.
+     * @param prevBltHistItem Previous baseline history item.
+     */
+    void onBaselineTopologyChanged(BaselineTopology blt, BaselineTopologyHistoryItem prevBltHistItem) throws IgniteCheckedException;
+
+    /**
+     * @param exchangeFuture Exchange future.
+     * @param hasMovingPartitions {@code True} if there are moving partitions.
+     */
+    void onExchangeFinishedOnCoordinator(IgniteInternalFuture exchangeFuture, boolean hasMovingPartitions);
+
+    /**
+     * @return {@code True} if partition evictions are allowed in current state.
+     */
+    boolean evictionsAllowed();
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/AtomicDataStructureProxy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/AtomicDataStructureProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/AtomicDataStructureProxy.java
index 8355210..2c336a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/AtomicDataStructureProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/AtomicDataStructureProxy.java
@@ -89,8 +89,8 @@ public abstract class AtomicDataStructureProxy<V extends AtomicDataStructureValu
      * @return Datastructure name.
      */
     public String name() {
-            return name;
-        }
+        return name;
+    }
 
     /**
      * @return Key value.

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
index acd8c11..99b2fa8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
@@ -17,7 +17,6 @@
 
 package org.apache.ignite.internal.processors.datastructures;
 
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
@@ -74,7 +73,6 @@ import org.apache.ignite.internal.util.typedef.CX1;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.A;
-import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.GPR;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -105,10 +103,10 @@ import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_REA
  */
 public final class DataStructuresProcessor extends GridProcessorAdapter implements IgniteChangeGlobalStateSupport {
     /** */
-    private static final String DEFAULT_DS_GROUP_NAME = "default-ds-group";
+    public static final String DEFAULT_VOLATILE_DS_GROUP_NAME = "default-volatile-ds-group";
 
     /** */
-    private static final String DEFAULT_VOLATILE_DS_GROUP_NAME = "default-volatile-ds-group";
+    private static final String DEFAULT_DS_GROUP_NAME = "default-ds-group";
 
     /** */
     private static final String DS_CACHE_NAME_PREFIX = "datastructures_";
@@ -172,13 +170,13 @@ public final class DataStructuresProcessor extends GridProcessorAdapter implemen
     }
 
     /** {@inheritDoc} */
-    @Override public void start() throws IgniteCheckedException {
+    @Override public void start() {
         ctx.event().addLocalEventListener(lsnr, EVT_NODE_LEFT, EVT_NODE_FAILED);
     }
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public void onKernalStart(boolean active) throws IgniteCheckedException {
+    @Override public void onKernalStart(boolean active) {
         if (ctx.config().isDaemon() || !active)
             return;
 
@@ -188,7 +186,14 @@ public final class DataStructuresProcessor extends GridProcessorAdapter implemen
     /**
      *
      */
-    private void onKernalStart0(){
+    public void onBeforeActivate() {
+        initLatch = new CountDownLatch(1);
+    }
+
+    /**
+     *
+     */
+    private void onKernalStart0() {
         initLatch.countDown();
     }
 
@@ -225,10 +230,14 @@ public final class DataStructuresProcessor extends GridProcessorAdapter implemen
                 ((GridCacheLockEx)ds).onStop();
         }
 
-        if (initLatch.getCount() > 0) {
+        CountDownLatch init0 = initLatch;
+
+        if (init0 != null && init0.getCount() > 0) {
             initFailed = true;
 
-            initLatch.countDown();
+            init0.countDown();
+
+            initLatch = null;
         }
 
         Iterator<Map.Entry<Integer, UUID>> iter = qryIdMap.entrySet().iterator();
@@ -245,22 +254,20 @@ public final class DataStructuresProcessor extends GridProcessorAdapter implemen
     }
 
     /** {@inheritDoc} */
-    @Override public void onActivate(GridKernalContext ctx) throws IgniteCheckedException {
+    @Override public void onActivate(GridKernalContext ctx) {
         if (log.isDebugEnabled())
-            log.debug("Activate data structure processor [nodeId=" + ctx.localNodeId() +
+            log.debug("Activating data structure processor [nodeId=" + ctx.localNodeId() +
                 " topVer=" + ctx.discovery().topologyVersionEx() + " ]");
 
         initFailed = false;
 
-        initLatch = new CountDownLatch(1);
-
         qryIdMap.clear();
 
         ctx.event().addLocalEventListener(lsnr, EVT_NODE_LEFT, EVT_NODE_FAILED);
 
-        onKernalStart0();
-
         restoreStructuresState(ctx);
+
+        onKernalStart0();
     }
 
     /**
@@ -284,12 +291,14 @@ public final class DataStructuresProcessor extends GridProcessorAdapter implemen
     @Override public void onDeActivate(GridKernalContext ctx) {
         if (log.isDebugEnabled())
             log.debug("DeActivate data structure processor [nodeId=" + ctx.localNodeId() +
-                " topVer=" + ctx.discovery().topologyVersionEx() + " ]");
+                ", topVer=" + ctx.discovery().topologyVersionEx() + "]");
 
         ctx.event().removeLocalEventListener(lsnr, EVT_NODE_LEFT, EVT_NODE_FAILED);
 
         onKernalStop(false);
 
+        initLatch = null;
+
         for (GridCacheRemovable v : dsMap.values()) {
             if (v instanceof IgniteChangeGlobalStateSupport)
                 ((IgniteChangeGlobalStateSupport)v).onDeActivate(ctx);
@@ -1070,9 +1079,14 @@ public final class DataStructuresProcessor extends GridProcessorAdapter implemen
      * Awaits for processor initialization.
      */
     private void awaitInitialization() {
-        if (initLatch.getCount() > 0) {
+        CountDownLatch latch0 = initLatch;
+
+        if (latch0 == null)
+            throw new IllegalStateException("Ignite cluster is not active");
+
+        if (latch0.getCount() > 0) {
             try {
-                U.await(initLatch);
+                U.await(latch0);
 
                 if (initFailed)
                     throw new IllegalStateException("Failed to initialize data structures processor.");

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index f53d4e2..1fd14f8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -2071,7 +2071,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
         if (qry.isLocal())
             throw new IgniteException("Local query is not supported without specific cache.");
 
-        if (!ctx.state().publicApiActiveState()) {
+        if (!ctx.state().publicApiActiveState(true)) {
             throw new IgniteException("Can not perform the operation because the cluster is inactive. Note, that " +
                 "the cluster is considered inactive by default if Ignite Persistent Store is used to let all the nodes " +
                 "join the cluster. To activate the cluster call Ignite.active(true).");

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
index 6236026..7bb13d9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
@@ -64,12 +64,12 @@ public class GridChangeStateCommandHandler extends GridRestCommandHandlerAdapter
 
         try {
             if (req.command().equals(CLUSTER_CURRENT_STATE)) {
-                Boolean currentState = ctx.state().publicApiActiveState();
+                Boolean currentState = ctx.state().publicApiActiveState(false);
 
                 res.setResponse(currentState);
             }
             else
-                ctx.state().changeGlobalState(req.active()).get();
+                ctx.grid().active(req.active());
 
             fut.onDone(res);
         }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 6df107b..f2b2781 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -238,9 +238,7 @@ public class GridServiceProcessor extends GridProcessorAdapter implements Ignite
             if (ctx.deploy().enabled())
                 ctx.cache().context().deploy().ignoreOwnership(true);
 
-            if (!ctx.clientNode()) {
-                assert serviceCache.context().affinityNode();
-
+            if (!ctx.clientNode() && serviceCache.context().affinityNode()) {
                 serviceCache.context().continuousQueries().executeInternalQuery(
                     new ServiceEntriesListener(), null, true, true, false
                 );
@@ -1790,8 +1788,7 @@ public class GridServiceProcessor extends GridProcessorAdapter implements Ignite
                                     try {
                                         svcName.set(dep.configuration().getName());
 
-                                        ctx.cache().internalCache(UTILITY_CACHE_NAME).context().affinity().
-                                            affinityReadyFuture(topVer).get();
+                                        ctx.cache().context().exchange().affinityReadyFuture(topVer).get();
 
                                         reassign(dep, topVer);
                                     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/subscription/GridInternalSubscriptionProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/subscription/GridInternalSubscriptionProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/subscription/GridInternalSubscriptionProcessor.java
new file mode 100644
index 0000000..6c0942a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/subscription/GridInternalSubscriptionProcessor.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.subscription;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.processors.GridProcessorAdapter;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Processor enables grid components to register listeners for events
+ * generated by other components on local node.
+ *
+ * It starts very first during node startup procedure so any components could use it.
+ *
+ * As only local events are supported, no network communication occurs during notification process.
+ */
+public class GridInternalSubscriptionProcessor extends GridProcessorAdapter {
+    /** */
+    private List<MetastorageLifecycleListener> metastorageListeners = new ArrayList<>();
+
+    /**
+     * @param ctx Kernal context.
+     */
+    public GridInternalSubscriptionProcessor(GridKernalContext ctx) {
+        super(ctx);
+    }
+
+    /** */
+    public void registerMetastorageListener(@NotNull MetastorageLifecycleListener metastorageListener) {
+        if (metastorageListener == null)
+            throw new NullPointerException("Metastorage subscriber should be not-null.");
+
+        metastorageListeners.add(metastorageListener);
+    }
+
+    /** */
+    public List<MetastorageLifecycleListener> getMetastorageSubscribers() {
+        return metastorageListeners;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFilteredClosableIterator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFilteredClosableIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFilteredClosableIterator.java
new file mode 100644
index 0000000..0ca795c
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFilteredClosableIterator.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util.lang;
+
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ *
+ */
+public abstract class GridFilteredClosableIterator<T> extends GridFilteredIterator<T> implements AutoCloseable {
+    /**
+     * @param it Closeable iterator.
+     */
+    protected GridFilteredClosableIterator(GridCloseableIterator<? extends T> it) {
+        super(it);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void close() throws IgniteCheckedException {
+        ((GridCloseableIterator)it).close();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFilteredIterator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFilteredIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFilteredIterator.java
index 8f074d3..452bdd3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFilteredIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFilteredIterator.java
@@ -25,7 +25,7 @@ import java.util.NoSuchElementException;
  */
 public abstract class GridFilteredIterator<T> implements Iterator<T> {
     /** */
-    private final Iterator<? extends T> it;
+    protected final Iterator<? extends T> it;
 
     /** */
     private boolean hasNext;

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
index 24b9d48..1c5a69a 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
@@ -168,6 +168,13 @@ public class GridFunc {
     /** */
     private static final IgniteClosure<ClusterNode, UUID> NODE2ID = new ClusterNodeGetIdClosure();
 
+    /** */
+    private static final IgniteClosure<ClusterNode, Object> NODE2CONSISTENTID = new IgniteClosure<ClusterNode, Object>() {
+        @Override public Object apply(ClusterNode node) {
+            return node.consistentId();
+        }
+    };
+
     /**
      * Gets predicate that evaluates to {@code true} only for given local node ID.
      *
@@ -324,6 +331,23 @@ public class GridFunc {
     }
 
     /**
+     * Convenient utility method that returns collection of node consistent IDs for a given
+     * collection of grid nodes.
+     * <p>
+     * Note that this method doesn't create a new collection but simply iterates
+     * over the input one.
+     *
+     * @param nodes Collection of grid nodes.
+     * @return Collection of node consistent IDs for given collection of grid nodes.
+     */
+    public static Collection<Object> nodeConsistentIds(@Nullable Collection<? extends ClusterNode> nodes) {
+        if (nodes == null || nodes.isEmpty())
+            return Collections.emptyList();
+
+        return F.viewReadOnly(nodes, NODE2CONSISTENTID);
+    }
+
+    /**
      * Gets random value from given collection.
      *
      * @param c Input collection (no {@code null} and not emtpy).

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineNode.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineNode.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineNode.java
new file mode 100644
index 0000000..b37a364
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineNode.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.baseline;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Map;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Data transfer object for {@link BaselineNode}.
+ */
+public class VisorBaselineNode extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private String consistentId;
+
+    /** */
+    private Map<String, Object> attrs;
+
+    /**
+     * Default constructor.
+     */
+    public VisorBaselineNode() {
+        // No-op.
+    }
+
+    /**
+     * Create data transfer object for baseline node.
+     *
+     * @param node Baseline node.
+     */
+    public VisorBaselineNode(BaselineNode node) {
+        consistentId = String.valueOf(node.consistentId());
+        attrs = node.attributes();
+    }
+
+    /**
+     * @return Node consistent ID.
+     */
+    public String getConsistentId() {
+        return consistentId;
+    }
+
+    /**
+     * @return Node attributes.
+     */
+    public Map<String, Object> getAttributes() {
+        return attrs;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeString(out, consistentId);
+        U.writeMap(out, attrs);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        consistentId = U.readString(in);
+        attrs = U.readMap(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorBaselineNode.class, this);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineOperation.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineOperation.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineOperation.java
new file mode 100644
index 0000000..b422417
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineOperation.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.baseline;
+
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Baseline operation types.
+ **/
+public enum VisorBaselineOperation {
+    /** Collect baseline information. */
+    COLLECT,
+    /** Add nodes to baseline. */
+    ADD,
+    /** Remove nodes from baseline. */
+    REMOVE,
+    /** Set new baseline. */
+    SET,
+    /** Set baseline for topology version. */
+    VERSION;
+
+    /** Enumerated values. */
+    private static final VisorBaselineOperation[] VALS = values();
+
+    /**
+     * Efficiently gets enumerated value from its ordinal.
+     *
+     * @param ord Ordinal value.
+     * @return Enumerated value or {@code null} if ordinal out of range.
+     */
+    @Nullable public static VisorBaselineOperation fromOrdinal(int ord) {
+        return ord >= 0 && ord < VALS.length ? VALS[ord] : null;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
new file mode 100644
index 0000000..56c2dd9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.baseline;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.cluster.IgniteClusterEx;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Task that will collect baseline topology information.
+ */
+@GridInternal
+public class VisorBaselineTask extends VisorOneNodeTask<VisorBaselineTaskArg, VisorBaselineTaskResult> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorBaselineJob job(VisorBaselineTaskArg arg) {
+        return new VisorBaselineJob(arg, debug);
+    }
+
+    /**
+     * Job that will collect baseline topology information.
+     */
+    private static class VisorBaselineJob extends VisorJob<VisorBaselineTaskArg, VisorBaselineTaskResult> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * @param arg Formal job argument.
+         * @param debug Debug flag.
+         */
+        private VisorBaselineJob(VisorBaselineTaskArg arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /**
+         * Collect baseline and server nodes.
+         *
+         * @return Baseline descriptor.
+         */
+        private VisorBaselineTaskResult collect() {
+            IgniteClusterEx cluster = ignite.cluster();
+
+            Collection<? extends BaselineNode> baseline = cluster.currentBaselineTopology();
+
+            Collection<? extends BaselineNode> servers = cluster.forServers().nodes();
+
+            return new VisorBaselineTaskResult(ignite.active(), cluster.topologyVersion(),
+                F.isEmpty(baseline) ? null : baseline, servers);
+        }
+
+        /**
+         * Set new baseline.
+         *
+         * @param baselineTop Collection of baseline node.
+         * @return Baseline descriptor.
+         */
+        private VisorBaselineTaskResult set0(Collection<BaselineNode> baselineTop) {
+            ignite.cluster().setBaselineTopology(baselineTop);
+
+            return collect();
+        }
+
+        /**
+         * @return Current baseline.
+         */
+        private Map<String, BaselineNode> currentBaseLine() {
+            Collection<BaselineNode> baseline = ignite.cluster().currentBaselineTopology();
+
+            Map<String, BaselineNode> nodes = new HashMap<>();
+
+            for (BaselineNode node : baseline)
+                nodes.put(node.consistentId().toString(), node);
+
+            return nodes;
+        }
+
+        /**
+         * @return Current server nodes.
+         */
+        private Map<String, BaselineNode> currentServers() {
+            Map<String, BaselineNode> nodes = new HashMap<>();
+
+            for (ClusterNode node : ignite.cluster().forServers().nodes())
+                nodes.put(node.consistentId().toString(), node);
+
+            return nodes;
+        }
+
+        /**
+         * Set new baseline.
+         *
+         * @param consistentIds Collection of consistent IDs to set.
+         * @return New baseline.
+         */
+        private VisorBaselineTaskResult set(List<String> consistentIds) {
+            Map<String, BaselineNode> servers = currentServers();
+
+            Collection<BaselineNode> baselineTop = new ArrayList<>();
+
+            for (String consistentId : consistentIds) {
+                BaselineNode node = servers.get(consistentId);
+
+                if (node == null)
+                    throw new IllegalStateException("Node not found for consistent ID: " + consistentId);
+
+                baselineTop.add(node);
+            }
+
+            return set0(baselineTop);
+        }
+
+        /**
+         * Add new nodes to baseline.
+         *
+         * @param consistentIds Collection of consistent IDs to add.
+         * @return New baseline.
+         */
+        private VisorBaselineTaskResult add(List<String> consistentIds) {
+            Map<String, BaselineNode> baseline = currentBaseLine();
+            Map<String, BaselineNode> servers = currentServers();
+
+            for (String consistentId : consistentIds) {
+                BaselineNode node = servers.get(consistentId);
+
+                if (node == null)
+                    throw new IllegalStateException("Node not found for consistent ID: " + consistentId);
+
+                baseline.put(consistentId, node);
+            }
+
+            return set0(baseline.values());
+        }
+
+        /**
+         * Remove nodes from baseline.
+         *
+         * @param consistentIds Collection of consistent IDs to remove.
+         * @return New baseline.
+         */
+        private VisorBaselineTaskResult remove(List<String> consistentIds) {
+            Map<String, BaselineNode> baseline = currentBaseLine();
+
+            for (String consistentId : consistentIds) {
+                BaselineNode node = baseline.remove(consistentId);
+
+                if (node == null)
+                    throw new IllegalStateException("Node not found for consistent ID: " + consistentId);
+            }
+
+            return set0(baseline.values());
+        }
+
+        /**
+         * Set baseline by topology version.
+         *
+         * @param targetVer Target topology version.
+         * @return New baseline.
+         */
+        private VisorBaselineTaskResult version(long targetVer) {
+            IgniteClusterEx cluster = ignite.cluster();
+
+            if (targetVer > cluster.topologyVersion())
+                throw new IllegalArgumentException("Topology version is ahead of time: " + targetVer);
+
+            cluster.setBaselineTopology(targetVer);
+
+            return collect();
+        }
+
+        /** {@inheritDoc} */
+        @Override protected VisorBaselineTaskResult run(@Nullable VisorBaselineTaskArg arg) throws IgniteException {
+            switch (arg.getOperation()) {
+                case ADD:
+                    return add(arg.getConsistentIds());
+
+                case REMOVE:
+                    return remove(arg.getConsistentIds());
+
+                case SET:
+                    return set(arg.getConsistentIds());
+
+                case VERSION:
+                    return version(arg.getTopologyVersion());
+
+                default:
+                    return collect();
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(VisorBaselineJob.class, this);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskArg.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskArg.java
new file mode 100644
index 0000000..092161e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskArg.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.baseline;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Argument for {@link VisorBaselineTask}.
+ */
+public class VisorBaselineTaskArg extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private VisorBaselineOperation op;
+
+    /** */
+    private long topVer;
+
+    /** */
+    private List<String> consistentIds;
+
+    /**
+     * Default constructor.
+     */
+    public VisorBaselineTaskArg() {
+        // No-op.
+    }
+
+    /**
+     * @param topVer Topology version.
+     */
+    public VisorBaselineTaskArg(VisorBaselineOperation op, long topVer, List<String> consistentIds) {
+        this.op = op;
+        this.topVer = topVer;
+        this.consistentIds = consistentIds;
+    }
+
+    /**
+     * @return Base line operation.
+     */
+    public VisorBaselineOperation getOperation() {
+        return op;
+    }
+
+    /**
+     * @return Topology version.
+     */
+    public long getTopologyVersion() {
+        return topVer;
+    }
+
+    /**
+     * @return Consistent IDs.
+     */
+    public List<String> getConsistentIds() {
+        return consistentIds;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeEnum(out, op);
+        out.writeLong(topVer);
+        U.writeCollection(out, consistentIds);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        op = VisorBaselineOperation.fromOrdinal(in.readByte());
+        topVer = in.readLong();
+        consistentIds = U.readList(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorBaselineTaskArg.class, this);
+    }
+}


[02/17] ignite git commit: IEP-4 Baseline topology for persistent caches (Phase 1) Contributed by: Dmitriy Govorukhin Dmitry Pavlov Eduard Shangareev Ily

Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListImplSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListImplSelfTest.java
new file mode 100644
index 0000000..80daff2
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListImplSelfTest.java
@@ -0,0 +1,565 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
+import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.CacheObjectContext;
+import org.apache.ignite.internal.processors.cache.CacheObjectValueContext;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.evict.NoOpPageEvictionTracker;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeListImpl;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public class CacheFreeListImplSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final int CPUS = Runtime.getRuntime().availableProcessors();
+
+    /** */
+    private static final long MB = 1024L * 1024L;
+
+    /** */
+    private PageMemory pageMem;
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        if (pageMem != null)
+            pageMem.stop();
+
+        pageMem = null;
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testInsertDeleteSingleThreaded_1024() throws Exception {
+        checkInsertDeleteSingleThreaded(1024);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testInsertDeleteSingleThreaded_2048() throws Exception {
+        checkInsertDeleteSingleThreaded(2048);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testInsertDeleteSingleThreaded_4096() throws Exception {
+        checkInsertDeleteSingleThreaded(4096);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testInsertDeleteSingleThreaded_8192() throws Exception {
+        checkInsertDeleteSingleThreaded(8192);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testInsertDeleteSingleThreaded_16384() throws Exception {
+        checkInsertDeleteSingleThreaded(16384);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testInsertDeleteMultiThreaded_1024() throws Exception {
+        checkInsertDeleteMultiThreaded(1024);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testInsertDeleteMultiThreaded_2048() throws Exception {
+        checkInsertDeleteMultiThreaded(2048);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testInsertDeleteMultiThreaded_4096() throws Exception {
+        checkInsertDeleteMultiThreaded(4096);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testInsertDeleteMultiThreaded_8192() throws Exception {
+        checkInsertDeleteMultiThreaded(8192);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testInsertDeleteMultiThreaded_16384() throws Exception {
+        checkInsertDeleteMultiThreaded(16384);
+    }
+
+    /**
+     * @param pageSize Page size.
+     * @throws Exception If failed.
+     */
+    protected void checkInsertDeleteMultiThreaded(final int pageSize) throws Exception {
+        final FreeList list = createFreeList(pageSize);
+
+        Random rnd = new Random();
+
+        final ConcurrentMap<Long, TestDataRow> stored = new ConcurrentHashMap<>();
+
+        for (int i = 0; i < 100; i++) {
+            int keySize = rnd.nextInt(pageSize * 3 / 2) + 10;
+            int valSize = rnd.nextInt(pageSize * 5 / 2) + 10;
+
+            TestDataRow row = new TestDataRow(keySize, valSize);
+
+            list.insertDataRow(row);
+
+            assertTrue(row.link() != 0L);
+
+            TestDataRow old = stored.put(row.link(), row);
+
+            assertNull(old);
+        }
+
+        final AtomicBoolean grow = new AtomicBoolean(true);
+
+        GridTestUtils.runMultiThreaded(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                Random rnd = ThreadLocalRandom.current();
+
+                for (int i = 0; i < 200_000; i++) {
+                    boolean grow0 = grow.get();
+
+                    if (grow0) {
+                        if (stored.size() > 20_000) {
+                            if (grow.compareAndSet(true, false))
+                                info("Shrink... [" + stored.size() + ']');
+
+                            grow0 = false;
+                        }
+                    }
+                    else {
+                        if (stored.size() < 1_000) {
+                            if (grow.compareAndSet(false, true))
+                                info("Grow... [" + stored.size() + ']');
+
+                            grow0 = true;
+                        }
+                    }
+
+                    boolean insert = rnd.nextInt(100) < 70 == grow0;
+
+                    if (insert) {
+                        int keySize = rnd.nextInt(pageSize * 3 / 2) + 10;
+                        int valSize = rnd.nextInt(pageSize * 3 / 2) + 10;
+
+                        TestDataRow row = new TestDataRow(keySize, valSize);
+
+                        list.insertDataRow(row);
+
+                        assertTrue(row.link() != 0L);
+
+                        TestDataRow old = stored.put(row.link(), row);
+
+                        assertNull(old);
+                    }
+                    else {
+                        while (true) {
+                            Iterator<TestDataRow> it = stored.values().iterator();
+
+                            if (it.hasNext()) {
+                                TestDataRow row = it.next();
+
+                                TestDataRow rmvd = stored.remove(row.link);
+
+                                if (rmvd != null) {
+                                    list.removeDataRowByLink(row.link);
+
+                                    break;
+                                }
+                            }
+                        }
+                    }
+                }
+
+                return null;
+            }
+        }, 8, "runner");
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    protected void checkInsertDeleteSingleThreaded(int pageSize) throws Exception {
+        FreeList list = createFreeList(pageSize);
+
+        Random rnd = new Random();
+
+        Map<Long, TestDataRow> stored = new HashMap<>();
+
+        for (int i = 0; i < 100; i++) {
+            int keySize = rnd.nextInt(pageSize * 3 / 2) + 10;
+            int valSize = rnd.nextInt(pageSize * 5 / 2) + 10;
+
+            TestDataRow row = new TestDataRow(keySize, valSize);
+
+            list.insertDataRow(row);
+
+            assertTrue(row.link() != 0L);
+
+            TestDataRow old = stored.put(row.link(), row);
+
+            assertNull(old);
+        }
+
+        boolean grow = true;
+
+        for (int i = 0; i < 1_000_000; i++) {
+            if (grow) {
+                if (stored.size() > 20_000) {
+                    grow = false;
+
+                    info("Shrink... [" + stored.size() + ']');
+                }
+            }
+            else {
+                if (stored.size() < 1_000) {
+                    grow = true;
+
+                    info("Grow... [" + stored.size() + ']');
+                }
+            }
+
+            boolean insert = rnd.nextInt(100) < 70 == grow;
+
+            if (insert) {
+                int keySize = rnd.nextInt(pageSize * 3 / 2) + 10;
+                int valSize = rnd.nextInt(pageSize * 3 / 2) + 10;
+
+                TestDataRow row = new TestDataRow(keySize, valSize);
+
+                list.insertDataRow(row);
+
+                assertTrue(row.link() != 0L);
+
+                TestDataRow old = stored.put(row.link(), row);
+
+                assertNull(old);
+            }
+            else {
+                Iterator<TestDataRow> it = stored.values().iterator();
+
+                if (it.hasNext()) {
+                    TestDataRow row = it.next();
+
+                    TestDataRow rmvd = stored.remove(row.link);
+
+                    assertTrue(rmvd == row);
+
+                    list.removeDataRowByLink(row.link);
+                }
+            }
+        }
+    }
+
+    /**
+     * @return Page memory.
+     */
+    protected PageMemory createPageMemory(int pageSize, DataRegionConfiguration plcCfg) throws Exception {
+        PageMemory pageMem = new PageMemoryNoStoreImpl(log,
+            new UnsafeMemoryProvider(log),
+            null,
+            pageSize,
+            plcCfg,
+            new DataRegionMetricsImpl(plcCfg),
+            true);
+
+        pageMem.start();
+
+        return pageMem;
+    }
+
+    /**
+     * @param pageSize Page size.
+     * @return Free list.
+     * @throws Exception If failed.
+     */
+    protected FreeList createFreeList(int pageSize) throws Exception {
+        DataRegionConfiguration plcCfg = new DataRegionConfiguration()
+            .setInitialSize(1024 * MB)
+            .setMaxSize(1024 * MB);
+
+        pageMem = createPageMemory(pageSize, plcCfg);
+
+        long metaPageId = pageMem.allocatePage(1, 1, PageIdAllocator.FLAG_DATA);
+
+        DataRegionMetricsImpl regionMetrics = new DataRegionMetricsImpl(plcCfg);
+
+        DataRegion dataRegion = new DataRegion(pageMem, plcCfg, regionMetrics, new NoOpPageEvictionTracker());
+
+        return new CacheFreeListImpl(1, "freelist", regionMetrics, dataRegion, null, null, metaPageId, true);
+    }
+
+    /**
+     *
+     */
+    private static class TestDataRow implements CacheDataRow {
+        /** */
+        private long link;
+
+        /** */
+        private TestCacheObject key;
+
+        /** */
+        private TestCacheObject val;
+
+        /** */
+        private GridCacheVersion ver;
+
+        /**
+         * @param keySize Key size.
+         * @param valSize Value size.
+         */
+        private TestDataRow(int keySize, int valSize) {
+            key = new TestCacheObject(keySize);
+            val = new TestCacheObject(valSize);
+            ver = new GridCacheVersion(keySize, valSize, 1);
+        }
+
+        /** {@inheritDoc} */
+        @Override public KeyCacheObject key() {
+            return key;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void key(KeyCacheObject key) {
+            this.key = (TestCacheObject)key;
+        }
+
+        /** {@inheritDoc} */
+        @Override public CacheObject value() {
+            return val;
+        }
+
+        /** {@inheritDoc} */
+        @Override public GridCacheVersion version() {
+            return ver;
+        }
+
+        /** {@inheritDoc} */
+        @Override public long expireTime() {
+            return 0;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int partition() {
+            return 0;
+        }
+
+        /** {@inheritDoc} */
+        @Override public long link() {
+            return link;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void link(long link) {
+            this.link = link;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hash() {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public int cacheId() {
+            return 0;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestCacheObject implements KeyCacheObject {
+        /** */
+        private byte[] data;
+
+        /**
+         * @param size Object size.
+         */
+        private TestCacheObject(int size) {
+            data = new byte[size];
+
+            Arrays.fill(data, (byte)size);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean internal() {
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int partition() {
+            return 0;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void partition(int part) {
+            assert false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public KeyCacheObject copy(int part) {
+            assert false;
+
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Nullable @Override public <T> T value(CacheObjectValueContext ctx, boolean cpy) {
+            return (T)data;
+        }
+
+        /** {@inheritDoc} */
+        @Override public byte[] valueBytes(CacheObjectValueContext ctx) throws IgniteCheckedException {
+            return data;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int valueBytesLength(CacheObjectContext ctx) throws IgniteCheckedException {
+            return data.length;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean putValue(ByteBuffer buf) throws IgniteCheckedException {
+            buf.put(data);
+
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int putValue(long addr) throws IgniteCheckedException {
+            PageUtils.putBytes(addr, 0, data);
+
+            return data.length;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean putValue(ByteBuffer buf, int off, int len) throws IgniteCheckedException {
+            buf.put(data, off, len);
+
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public byte cacheObjectType() {
+            return 42;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean isPlatformType() {
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public CacheObject prepareForCache(CacheObjectContext ctx) {
+            assert false;
+
+            return this;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void finishUnmarshal(CacheObjectValueContext ctx, ClassLoader ldr)
+            throws IgniteCheckedException {
+            assert false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void prepareMarshal(CacheObjectValueContext ctx) throws IgniteCheckedException {
+            assert false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+            assert false;
+
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+            assert false;
+
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public short directType() {
+            assert false;
+
+            return 0;
+        }
+
+        /** {@inheritDoc} */
+        @Override public byte fieldsCount() {
+            assert false;
+
+            return 0;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onAckReceived() {
+            assert false;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/database/FreeListImplSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/FreeListImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/FreeListImplSelfTest.java
deleted file mode 100644
index 72a1d81..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/FreeListImplSelfTest.java
+++ /dev/null
@@ -1,565 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.database;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.configuration.DataRegionConfiguration;
-import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
-import org.apache.ignite.internal.pagemem.PageIdAllocator;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
-import org.apache.ignite.internal.processors.cache.CacheObject;
-import org.apache.ignite.internal.processors.cache.CacheObjectContext;
-import org.apache.ignite.internal.processors.cache.CacheObjectValueContext;
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
-import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
-import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
-import org.apache.ignite.internal.processors.cache.persistence.evict.NoOpPageEvictionTracker;
-import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
-import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl;
-import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
-import org.apache.ignite.plugin.extensions.communication.MessageReader;
-import org.apache.ignite.plugin.extensions.communication.MessageWriter;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.jetbrains.annotations.Nullable;
-
-/**
- *
- */
-public class FreeListImplSelfTest extends GridCommonAbstractTest {
-    /** */
-    private static final int CPUS = Runtime.getRuntime().availableProcessors();
-
-    /** */
-    private static final long MB = 1024L * 1024L;
-
-    /** */
-    private PageMemory pageMem;
-
-    /** {@inheritDoc} */
-    @Override protected void afterTest() throws Exception {
-        super.afterTest();
-
-        if (pageMem != null)
-            pageMem.stop();
-
-        pageMem = null;
-    }
-
-    /**
-     * @throws Exception if failed.
-     */
-    public void testInsertDeleteSingleThreaded_1024() throws Exception {
-        checkInsertDeleteSingleThreaded(1024);
-    }
-
-    /**
-     * @throws Exception if failed.
-     */
-    public void testInsertDeleteSingleThreaded_2048() throws Exception {
-        checkInsertDeleteSingleThreaded(2048);
-    }
-
-    /**
-     * @throws Exception if failed.
-     */
-    public void testInsertDeleteSingleThreaded_4096() throws Exception {
-        checkInsertDeleteSingleThreaded(4096);
-    }
-
-    /**
-     * @throws Exception if failed.
-     */
-    public void testInsertDeleteSingleThreaded_8192() throws Exception {
-        checkInsertDeleteSingleThreaded(8192);
-    }
-
-    /**
-     * @throws Exception if failed.
-     */
-    public void testInsertDeleteSingleThreaded_16384() throws Exception {
-        checkInsertDeleteSingleThreaded(16384);
-    }
-
-    /**
-     * @throws Exception if failed.
-     */
-    public void testInsertDeleteMultiThreaded_1024() throws Exception {
-        checkInsertDeleteMultiThreaded(1024);
-    }
-
-    /**
-     * @throws Exception if failed.
-     */
-    public void testInsertDeleteMultiThreaded_2048() throws Exception {
-        checkInsertDeleteMultiThreaded(2048);
-    }
-
-    /**
-     * @throws Exception if failed.
-     */
-    public void testInsertDeleteMultiThreaded_4096() throws Exception {
-        checkInsertDeleteMultiThreaded(4096);
-    }
-
-    /**
-     * @throws Exception if failed.
-     */
-    public void testInsertDeleteMultiThreaded_8192() throws Exception {
-        checkInsertDeleteMultiThreaded(8192);
-    }
-
-    /**
-     * @throws Exception if failed.
-     */
-    public void testInsertDeleteMultiThreaded_16384() throws Exception {
-        checkInsertDeleteMultiThreaded(16384);
-    }
-
-    /**
-     * @param pageSize Page size.
-     * @throws Exception If failed.
-     */
-    protected void checkInsertDeleteMultiThreaded(final int pageSize) throws Exception {
-        final FreeList list = createFreeList(pageSize);
-
-        Random rnd = new Random();
-
-        final ConcurrentMap<Long, TestDataRow> stored = new ConcurrentHashMap<>();
-
-        for (int i = 0; i < 100; i++) {
-            int keySize = rnd.nextInt(pageSize * 3 / 2) + 10;
-            int valSize = rnd.nextInt(pageSize * 5 / 2) + 10;
-
-            TestDataRow row = new TestDataRow(keySize, valSize);
-
-            list.insertDataRow(row);
-
-            assertTrue(row.link() != 0L);
-
-            TestDataRow old = stored.put(row.link(), row);
-
-            assertNull(old);
-        }
-
-        final AtomicBoolean grow = new AtomicBoolean(true);
-
-        GridTestUtils.runMultiThreaded(new Callable<Object>() {
-            @Override public Object call() throws Exception {
-                Random rnd = ThreadLocalRandom.current();
-
-                for (int i = 0; i < 200_000; i++) {
-                    boolean grow0 = grow.get();
-
-                    if (grow0) {
-                        if (stored.size() > 20_000) {
-                            if (grow.compareAndSet(true, false))
-                                info("Shrink... [" + stored.size() + ']');
-
-                            grow0 = false;
-                        }
-                    }
-                    else {
-                        if (stored.size() < 1_000) {
-                            if (grow.compareAndSet(false, true))
-                                info("Grow... [" + stored.size() + ']');
-
-                            grow0 = true;
-                        }
-                    }
-
-                    boolean insert = rnd.nextInt(100) < 70 == grow0;
-
-                    if (insert) {
-                        int keySize = rnd.nextInt(pageSize * 3 / 2) + 10;
-                        int valSize = rnd.nextInt(pageSize * 3 / 2) + 10;
-
-                        TestDataRow row = new TestDataRow(keySize, valSize);
-
-                        list.insertDataRow(row);
-
-                        assertTrue(row.link() != 0L);
-
-                        TestDataRow old = stored.put(row.link(), row);
-
-                        assertNull(old);
-                    }
-                    else {
-                        while (true) {
-                            Iterator<TestDataRow> it = stored.values().iterator();
-
-                            if (it.hasNext()) {
-                                TestDataRow row = it.next();
-
-                                TestDataRow rmvd = stored.remove(row.link);
-
-                                if (rmvd != null) {
-                                    list.removeDataRowByLink(row.link);
-
-                                    break;
-                                }
-                            }
-                        }
-                    }
-                }
-
-                return null;
-            }
-        }, 8, "runner");
-    }
-
-    /**
-     * @throws Exception if failed.
-     */
-    protected void checkInsertDeleteSingleThreaded(int pageSize) throws Exception {
-        FreeList list = createFreeList(pageSize);
-
-        Random rnd = new Random();
-
-        Map<Long, TestDataRow> stored = new HashMap<>();
-
-        for (int i = 0; i < 100; i++) {
-            int keySize = rnd.nextInt(pageSize * 3 / 2) + 10;
-            int valSize = rnd.nextInt(pageSize * 5 / 2) + 10;
-
-            TestDataRow row = new TestDataRow(keySize, valSize);
-
-            list.insertDataRow(row);
-
-            assertTrue(row.link() != 0L);
-
-            TestDataRow old = stored.put(row.link(), row);
-
-            assertNull(old);
-        }
-
-        boolean grow = true;
-
-        for (int i = 0; i < 1_000_000; i++) {
-            if (grow) {
-                if (stored.size() > 20_000) {
-                    grow = false;
-
-                    info("Shrink... [" + stored.size() + ']');
-                }
-            }
-            else {
-                if (stored.size() < 1_000) {
-                    grow = true;
-
-                    info("Grow... [" + stored.size() + ']');
-                }
-            }
-
-            boolean insert = rnd.nextInt(100) < 70 == grow;
-
-            if (insert) {
-                int keySize = rnd.nextInt(pageSize * 3 / 2) + 10;
-                int valSize = rnd.nextInt(pageSize * 3 / 2) + 10;
-
-                TestDataRow row = new TestDataRow(keySize, valSize);
-
-                list.insertDataRow(row);
-
-                assertTrue(row.link() != 0L);
-
-                TestDataRow old = stored.put(row.link(), row);
-
-                assertNull(old);
-            }
-            else {
-                Iterator<TestDataRow> it = stored.values().iterator();
-
-                if (it.hasNext()) {
-                    TestDataRow row = it.next();
-
-                    TestDataRow rmvd = stored.remove(row.link);
-
-                    assertTrue(rmvd == row);
-
-                    list.removeDataRowByLink(row.link);
-                }
-            }
-        }
-    }
-
-    /**
-     * @return Page memory.
-     */
-    protected PageMemory createPageMemory(int pageSize, DataRegionConfiguration plcCfg) throws Exception {
-        PageMemory pageMem = new PageMemoryNoStoreImpl(log,
-            new UnsafeMemoryProvider(log),
-            null,
-            pageSize,
-            plcCfg,
-            new DataRegionMetricsImpl(plcCfg),
-            true);
-
-        pageMem.start();
-
-        return pageMem;
-    }
-
-    /**
-     * @param pageSize Page size.
-     * @return Free list.
-     * @throws Exception If failed.
-     */
-    protected FreeList createFreeList(int pageSize) throws Exception {
-        DataRegionConfiguration plcCfg = new DataRegionConfiguration()
-            .setInitialSize(1024 * MB)
-            .setMaxSize(1024 * MB);
-
-        pageMem = createPageMemory(pageSize, plcCfg);
-
-        long metaPageId = pageMem.allocatePage(1, 1, PageIdAllocator.FLAG_DATA);
-
-        DataRegionMetricsImpl metrics = new DataRegionMetricsImpl(plcCfg);
-
-        DataRegion memPlc = new DataRegion(pageMem, plcCfg, metrics, new NoOpPageEvictionTracker());
-
-        return new FreeListImpl(1, "freelist", metrics, memPlc, null, null, metaPageId, true);
-    }
-
-    /**
-     *
-     */
-    private static class TestDataRow implements CacheDataRow {
-        /** */
-        private long link;
-
-        /** */
-        private TestCacheObject key;
-
-        /** */
-        private TestCacheObject val;
-
-        /** */
-        private GridCacheVersion ver;
-
-        /**
-         * @param keySize Key size.
-         * @param valSize Value size.
-         */
-        private TestDataRow(int keySize, int valSize) {
-            key = new TestCacheObject(keySize);
-            val = new TestCacheObject(valSize);
-            ver = new GridCacheVersion(keySize, valSize, 1);
-        }
-
-        /** {@inheritDoc} */
-        @Override public KeyCacheObject key() {
-            return key;
-        }
-
-        /** {@inheritDoc} */
-        @Override public void key(KeyCacheObject key) {
-            this.key = (TestCacheObject)key;
-        }
-
-        /** {@inheritDoc} */
-        @Override public CacheObject value() {
-            return val;
-        }
-
-        /** {@inheritDoc} */
-        @Override public GridCacheVersion version() {
-            return ver;
-        }
-
-        /** {@inheritDoc} */
-        @Override public long expireTime() {
-            return 0;
-        }
-
-        /** {@inheritDoc} */
-        @Override public int partition() {
-            return 0;
-        }
-
-        /** {@inheritDoc} */
-        @Override public long link() {
-            return link;
-        }
-
-        /** {@inheritDoc} */
-        @Override public void link(long link) {
-            this.link = link;
-        }
-
-        /** {@inheritDoc} */
-        @Override public int hash() {
-            throw new UnsupportedOperationException();
-        }
-
-        /** {@inheritDoc} */
-        @Override public int cacheId() {
-            return 0;
-        }
-    }
-
-    /**
-     *
-     */
-    private static class TestCacheObject implements KeyCacheObject {
-        /** */
-        private byte[] data;
-
-        /**
-         * @param size Object size.
-         */
-        private TestCacheObject(int size) {
-            data = new byte[size];
-
-            Arrays.fill(data, (byte)size);
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean internal() {
-            return false;
-        }
-
-        /** {@inheritDoc} */
-        @Override public int partition() {
-            return 0;
-        }
-
-        /** {@inheritDoc} */
-        @Override public void partition(int part) {
-            assert false;
-        }
-
-        /** {@inheritDoc} */
-        @Override public KeyCacheObject copy(int part) {
-            assert false;
-
-            return null;
-        }
-
-        /** {@inheritDoc} */
-        @Nullable @Override public <T> T value(CacheObjectValueContext ctx, boolean cpy) {
-            return (T)data;
-        }
-
-        /** {@inheritDoc} */
-        @Override public byte[] valueBytes(CacheObjectValueContext ctx) throws IgniteCheckedException {
-            return data;
-        }
-
-        /** {@inheritDoc} */
-        @Override public int valueBytesLength(CacheObjectContext ctx) throws IgniteCheckedException {
-            return data.length;
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean putValue(ByteBuffer buf) throws IgniteCheckedException {
-            buf.put(data);
-
-            return true;
-        }
-
-        /** {@inheritDoc} */
-        @Override public int putValue(long addr) throws IgniteCheckedException {
-            PageUtils.putBytes(addr, 0, data);
-
-            return data.length;
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean putValue(ByteBuffer buf, int off, int len) throws IgniteCheckedException {
-            buf.put(data, off, len);
-
-            return true;
-        }
-
-        /** {@inheritDoc} */
-        @Override public byte cacheObjectType() {
-            return 42;
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean isPlatformType() {
-            return false;
-        }
-
-        /** {@inheritDoc} */
-        @Override public CacheObject prepareForCache(CacheObjectContext ctx) {
-            assert false;
-
-            return this;
-        }
-
-        /** {@inheritDoc} */
-        @Override public void finishUnmarshal(CacheObjectValueContext ctx, ClassLoader ldr)
-            throws IgniteCheckedException {
-            assert false;
-        }
-
-        /** {@inheritDoc} */
-        @Override public void prepareMarshal(CacheObjectValueContext ctx) throws IgniteCheckedException {
-            assert false;
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
-            assert false;
-
-            return false;
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
-            assert false;
-
-            return false;
-        }
-
-        /** {@inheritDoc} */
-        @Override public short directType() {
-            assert false;
-
-            return 0;
-        }
-
-        /** {@inheritDoc} */
-        @Override public byte fieldsCount() {
-            assert false;
-
-            return 0;
-        }
-
-        /** {@inheritDoc} */
-        @Override public void onAckReceived() {
-            assert false;
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetWithCacheStoreTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetWithCacheStoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetWithCacheStoreTest.java
new file mode 100644
index 0000000..c76b6e2
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetWithCacheStoreTest.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import javax.cache.Cache;
+import javax.cache.integration.CacheLoaderException;
+import javax.cache.integration.CacheWriterException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.store.CacheStoreAdapter;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.IgniteReflectionFactory;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+
+/**
+ *
+ */
+public class IgniteDbPutGetWithCacheStoreTest extends GridCommonAbstractTest {
+    /** */
+    private static Map<Object, Object> storeMap = new ConcurrentHashMap<>();
+
+    /** */
+    private static final String CACHE_NAME = "cache";
+
+    /** */
+    private CacheAtomicityMode atomicityMode;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        DataStorageConfiguration dbCfg = new DataStorageConfiguration();
+
+        dbCfg
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setMaxSize(512 * 1024 * 1024)
+                    .setPersistenceEnabled(true))
+            .setWalMode(WALMode.LOG_ONLY);
+
+        cfg.setDataStorageConfiguration(dbCfg);
+
+        CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>(CACHE_NAME)
+            .setCacheStoreFactory(new IgniteReflectionFactory<>(TestStore.class))
+            .setAtomicityMode(atomicityMode)
+            .setBackups(1)
+            .setWriteThrough(true)
+            .setReadThrough(true);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        storeMap.clear();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        storeMap.clear();
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testWriteThrough() throws Exception {
+        checkWriteThrough(ATOMIC);
+        checkWriteThrough(TRANSACTIONAL);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testReadThrough() throws Exception {
+        checkReadThrough(ATOMIC);
+        checkReadThrough(TRANSACTIONAL);
+    }
+
+    /**
+     * @param atomicityMode Atomicity mode.
+     * @throws Exception if failed.
+     */
+    private void checkWriteThrough(CacheAtomicityMode atomicityMode) throws Exception {
+        this.atomicityMode = atomicityMode;
+
+        IgniteEx ig = startGrid(0);
+
+        try {
+            ig.active(true);
+
+            for (int i = 0; i < 2000; i++)
+                ig.cache(CACHE_NAME).put(i, i);
+
+            assertEquals(2000, storeMap.size());
+
+            stopAllGrids();
+
+            storeMap.clear();
+
+            ig = startGrid(0);
+
+            ig.active(true);
+
+            for (int i = 0; i < 2000; i++)
+                assertEquals(i, ig.cache(CACHE_NAME).get(i));
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * @param atomicityMode Atomicity mode.
+     * @throws Exception if failed.
+     */
+    private void checkReadThrough(CacheAtomicityMode atomicityMode) throws Exception {
+        this.atomicityMode = atomicityMode;
+
+        IgniteEx ig = startGrid(0);
+
+        try {
+            ig.active(true);
+
+            for (int i = 0; i < 2000; i++)
+                storeMap.put(i, i);
+
+            for (int i = 0; i < 2000; i++)
+                assertEquals(i, ig.cache(CACHE_NAME).get(i));
+
+            stopAllGrids();
+
+            storeMap.clear();
+
+            ig = startGrid(0);
+
+            ig.active(true);
+
+            for (int i = 0; i < 2000; i++)
+                assertEquals(i, ig.cache(CACHE_NAME).get(i));
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     *
+     */
+    public static class TestStore extends CacheStoreAdapter<Object, Object> {
+        /** {@inheritDoc} */
+        @Override public Object load(Object key) throws CacheLoaderException {
+            return storeMap.get(key);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(Cache.Entry<?, ?> entry) throws CacheWriterException {
+            storeMap.put(entry.getKey(), entry.getValue());
+        }
+
+        /** {@inheritDoc} */
+        @Override public void delete(Object key) throws CacheWriterException {
+            storeMap.remove(key);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java
new file mode 100644
index 0000000..04bea54
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
+import org.apache.ignite.internal.processors.cache.persistence.IndexStorageImpl;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.RootPage;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IndexStorageSelfTest extends GridCommonAbstractTest {
+    /** Make sure page is small enough to trigger multiple pages in a linked list. */
+    private static final int PAGE_SIZE = 1024;
+
+    /** */
+    private static File allocationPath;
+
+    /** */
+    private static final char[] ALPHABET = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ".toCharArray();
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        allocationPath = U.resolveWorkDirectory(U.defaultWorkDirectory(), "pagemem", false);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testMetaIndexAllocation() throws Exception {
+        metaAllocation();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void metaAllocation() throws Exception {
+        PageMemory mem = memory(true);
+
+        int[] cacheIds = new int[]{1, "partitioned".hashCode(), "replicated".hashCode()};
+
+        Map<Integer, Map<String, RootPage>> allocatedIdxs = new HashMap<>();
+
+        mem.start();
+
+        try {
+            final Map<Integer, IndexStorageImpl> storeMap = new HashMap<>();
+
+            for (int i = 0; i < 1_000; i++) {
+                int cacheId = cacheIds[i % cacheIds.length];
+
+                Map<String, RootPage> idxMap = allocatedIdxs.get(cacheId);
+
+                if (idxMap == null) {
+                    idxMap = new HashMap<>();
+
+                    allocatedIdxs.put(cacheId, idxMap);
+                }
+
+                String idxName;
+
+                do {
+                    idxName = randomName();
+                } while (idxMap.containsKey(idxName));
+
+                IndexStorageImpl metaStore = storeMap.get(cacheId);
+
+                if (metaStore == null) {
+                    metaStore = new IndexStorageImpl(mem, null, new AtomicLong(), cacheId,
+                        PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX,
+                        null, mem.allocatePage(cacheId, PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX), true);
+
+                    storeMap.put(cacheId, metaStore);
+                }
+
+                final RootPage rootPage = metaStore.getOrAllocateForTree(idxName);
+
+                assertTrue(rootPage.isAllocated());
+
+                idxMap.put(idxName, rootPage);
+            }
+
+            for (int cacheId : cacheIds) {
+                Map<String, RootPage> idxMap = allocatedIdxs.get(cacheId);
+
+                for (Map.Entry<String, RootPage> entry : idxMap.entrySet()) {
+                    String idxName = entry.getKey();
+                    FullPageId rootPageId = entry.getValue().pageId();
+
+                    final RootPage rootPage = storeMap.get(cacheId).getOrAllocateForTree(idxName);
+
+                    assertEquals("Invalid root page ID restored [cacheId=" + cacheId + ", idxName=" + idxName + ']',
+                        rootPageId, rootPage.pageId());
+
+                    assertFalse("Root page already allocated [cacheId=" + cacheId + ", idxName=" + idxName + ']',
+                        rootPage.isAllocated());
+                }
+            }
+        }
+        finally {
+            mem.stop();
+        }
+    }
+
+    /**
+     * @return Random name.
+     */
+    private static String randomName() {
+        StringBuilder sb = new StringBuilder();
+
+        Random rnd = ThreadLocalRandom.current();
+
+        int size = rnd.nextInt(25) + 1;
+
+        for (int i = 0; i < size; i++)
+            sb.append(ALPHABET[rnd.nextInt(ALPHABET.length)]);
+
+        return sb.toString();
+    }
+
+    /**
+     * @param clean Clean flag. If {@code true}, will clean previous memory state and allocate
+     *      new empty page memory.
+     * @return Page memory instance.
+     */
+    protected PageMemory memory(boolean clean) throws Exception {
+        DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), allocationPath);
+
+        DataRegionConfiguration plcCfg = new DataRegionConfiguration()
+            .setMaxSize(30 * 1024 * 1024).setInitialSize(30 * 1024 * 1024);
+
+        return new PageMemoryNoStoreImpl(
+            log,
+            provider,
+            null,
+            PAGE_SIZE,
+            plcCfg,
+            new DataRegionMetricsImpl(plcCfg),
+            true);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java
deleted file mode 100644
index 880e37e..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.database;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.ignite.configuration.DataRegionConfiguration;
-import org.apache.ignite.internal.mem.DirectMemoryProvider;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageIdAllocator;
-import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
-import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
-import org.apache.ignite.internal.processors.cache.persistence.MetadataStorage;
-import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.persistence.RootPage;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class MetadataStorageSelfTest extends GridCommonAbstractTest {
-    /** Make sure page is small enough to trigger multiple pages in a linked list. */
-    private static final int PAGE_SIZE = 1024;
-
-    /** */
-    private static File allocationPath;
-
-    /** */
-    private static final char[] ALPHABET = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ".toCharArray();
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        allocationPath = U.resolveWorkDirectory(U.defaultWorkDirectory(), "pagemem", false);
-    }
-
-    /**
-     * @throws Exception if failed.
-     */
-    public void testMetaIndexAllocation() throws Exception {
-        metaAllocation();
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    private void metaAllocation() throws Exception {
-        PageMemory mem = memory(true);
-
-        int[] cacheIds = new int[]{1, "partitioned".hashCode(), "replicated".hashCode()};
-
-        Map<Integer, Map<String, RootPage>> allocatedIdxs = new HashMap<>();
-
-        mem.start();
-
-        try {
-            final Map<Integer, MetadataStorage> storeMap = new HashMap<>();
-
-            for (int i = 0; i < 1_000; i++) {
-                int cacheId = cacheIds[i % cacheIds.length];
-
-                Map<String, RootPage> idxMap = allocatedIdxs.get(cacheId);
-
-                if (idxMap == null) {
-                    idxMap = new HashMap<>();
-
-                    allocatedIdxs.put(cacheId, idxMap);
-                }
-
-                String idxName;
-
-                do {
-                    idxName = randomName();
-                } while (idxMap.containsKey(idxName));
-
-                MetadataStorage metaStore = storeMap.get(cacheId);
-
-                if (metaStore == null) {
-                    metaStore = new MetadataStorage(mem, null, new AtomicLong(), cacheId,
-                        PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX,
-                        null, mem.allocatePage(cacheId, PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX), true);
-
-                    storeMap.put(cacheId, metaStore);
-                }
-
-                final RootPage rootPage = metaStore.getOrAllocateForTree(idxName);
-
-                assertTrue(rootPage.isAllocated());
-
-                idxMap.put(idxName, rootPage);
-            }
-
-            for (int cacheId : cacheIds) {
-                Map<String, RootPage> idxMap = allocatedIdxs.get(cacheId);
-
-                for (Map.Entry<String, RootPage> entry : idxMap.entrySet()) {
-                    String idxName = entry.getKey();
-                    FullPageId rootPageId = entry.getValue().pageId();
-
-                    final RootPage rootPage = storeMap.get(cacheId).getOrAllocateForTree(idxName);
-
-                    assertEquals("Invalid root page ID restored [cacheId=" + cacheId + ", idxName=" + idxName + ']',
-                        rootPageId, rootPage.pageId());
-
-                    assertFalse("Root page already allocated [cacheId=" + cacheId + ", idxName=" + idxName + ']',
-                        rootPage.isAllocated());
-                }
-            }
-        }
-        finally {
-            mem.stop();
-        }
-    }
-
-    /**
-     * @return Random name.
-     */
-    private static String randomName() {
-        StringBuilder sb = new StringBuilder();
-
-        Random rnd = ThreadLocalRandom.current();
-
-        int size = rnd.nextInt(25) + 1;
-
-        for (int i = 0; i < size; i++)
-            sb.append(ALPHABET[rnd.nextInt(ALPHABET.length)]);
-
-        return sb.toString();
-    }
-
-    /**
-     * @param clean Clean flag. If {@code true}, will clean previous memory state and allocate
-     *      new empty page memory.
-     * @return Page memory instance.
-     */
-    protected PageMemory memory(boolean clean) throws Exception {
-        DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), allocationPath);
-
-        DataRegionConfiguration plcCfg = new DataRegionConfiguration()
-            .setMaxSize(30 * 1024 * 1024).setInitialSize(30 * 1024 * 1024);
-
-        return new PageMemoryNoStoreImpl(
-            log,
-            provider,
-            null,
-            PAGE_SIZE,
-            plcCfg,
-            new DataRegionMetricsImpl(plcCfg),
-            true);
-    }
-}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
index 93cd911..ca05288 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
@@ -61,6 +61,7 @@ import org.apache.ignite.spi.IgnitePortProtocol;
 import org.apache.ignite.spi.IgniteSpiContext;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.IgniteSpiTimeoutObject;
+import org.apache.ignite.spi.discovery.DiscoveryDataBag;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
@@ -541,6 +542,11 @@ public class GridSpiTestContext implements IgniteSpiContext {
     }
 
     /** {@inheritDoc} */
+    @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode node, DiscoveryDataBag discoData) {
+        return null;
+    }
+
+    /** {@inheritDoc} */
     @Override public Collection<SecuritySubject> authenticatedSubjects() {
         return Collections.emptyList();
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
index 53ca516..91a1144 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
@@ -1973,4 +1973,13 @@ public final class GridTestUtils {
     public static void deleteDbFiles() throws Exception {
         deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
     }
+
+    /**
+     * @param node Node.
+     * @param topVer Ready exchange version to wait for before trying to merge exchanges.
+     */
+    public static void mergeExchangeWaitVersion(Ignite node, long topVer) {
+        ((IgniteEx)node).context().cache().context().exchange().mergeExchangesTestWaitVersion(
+            new AffinityTopologyVersion(topVer, 0));
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index f5e8eef..74a0de9 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -60,8 +60,8 @@ import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityFunctionContextImpl;
@@ -85,6 +85,9 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCach
 import org.apache.ignite.internal.processors.cache.local.GridLocalCache;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager;
+import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
+import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
+import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTask;
 import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
@@ -106,7 +109,6 @@ import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.cache.CacheMode.LOCAL;
-import static org.apache.ignite.cache.CacheMode.PARTITIONED;
 import static org.apache.ignite.cache.CacheRebalanceMode.NONE;
 import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isNearEnabled;
 import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
@@ -769,6 +771,25 @@ public abstract class GridCommonAbstractTest extends GridAbstractTest {
     }
 
     /**
+     * Compares checksums between primary and backup partitions of specified caches.
+     * Works properly only on idle cluster - there may be false positive conflict reports if data in cluster is being
+     * concurrently updated.
+     *
+     * @param ig Ignite instance.
+     * @param cacheNames Cache names (if null, all user caches will be verified).
+     * @throws IgniteCheckedException If checksum conflict has been found.
+     */
+    protected void verifyBackupPartitions(Ignite ig, Set<String> cacheNames) throws IgniteCheckedException {
+        Map<PartitionKey, List<PartitionHashRecord>> conflicts = ig.compute().execute(
+            new VerifyBackupPartitionsTask(), cacheNames);
+
+        if (!conflicts.isEmpty()) {
+            throw new IgniteCheckedException("Partition checksums are different for backups " +
+                "of the following partitions: " + conflicts.keySet());
+        }
+    }
+
+    /**
      * @param top Topology.
      * @param topVer Version to wait for.
      * @throws Exception If failed.

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java
index 534b07f..075ab91 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java
@@ -26,6 +26,7 @@ import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCluster;
 import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.BaselineNode;
 import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.cluster.ClusterMetrics;
 import org.apache.ignite.cluster.ClusterNode;
@@ -314,6 +315,31 @@ public class IgniteClusterProcessProxy implements IgniteClusterEx {
         throw new UnsupportedOperationException("Operation is not supported yet.");
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean active() {
+        throw new UnsupportedOperationException("Operation is not supported yet.");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void active(boolean active) {
+        throw new UnsupportedOperationException("Operation is not supported yet.");
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Collection<BaselineNode> currentBaselineTopology() {
+        throw new UnsupportedOperationException("Operation is not supported yet.");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void setBaselineTopology(Collection<? extends BaselineNode> baselineTop) {
+        throw new UnsupportedOperationException("Operation is not supported yet.");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void setBaselineTopology(long topVer) {
+        throw new UnsupportedOperationException("Operation is not supported yet.");
+    }
+
     /**
      *
      */

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index 2cc2047..1b4e2da 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -54,9 +54,9 @@ import org.apache.ignite.internal.processors.continuous.GridMessageListenSelfTes
 import org.apache.ignite.internal.processors.database.BPlusTreeFakeReuseSelfTest;
 import org.apache.ignite.internal.processors.database.BPlusTreeReuseSelfTest;
 import org.apache.ignite.internal.processors.database.BPlusTreeSelfTest;
-import org.apache.ignite.internal.processors.database.FreeListImplSelfTest;
+import org.apache.ignite.internal.processors.database.CacheFreeListImplSelfTest;
+import org.apache.ignite.internal.processors.database.IndexStorageSelfTest;
 import org.apache.ignite.internal.processors.database.DataRegionMetricsSelfTest;
-import org.apache.ignite.internal.processors.database.MetadataStorageSelfTest;
 import org.apache.ignite.internal.processors.database.SwapPathConstructionSelfTest;
 import org.apache.ignite.internal.processors.odbc.OdbcConfigurationValidationSelfTest;
 import org.apache.ignite.internal.processors.odbc.OdbcEscapeSequenceSelfTest;
@@ -176,8 +176,8 @@ public class IgniteBasicTestSuite extends TestSuite {
         suite.addTestSuite(BPlusTreeSelfTest.class);
         suite.addTestSuite(BPlusTreeFakeReuseSelfTest.class);
         suite.addTestSuite(BPlusTreeReuseSelfTest.class);
-        suite.addTestSuite(MetadataStorageSelfTest.class);
-        suite.addTestSuite(FreeListImplSelfTest.class);
+        suite.addTestSuite(IndexStorageSelfTest.class);
+        suite.addTestSuite(CacheFreeListImplSelfTest.class);
         suite.addTestSuite(DataRegionMetricsSelfTest.class);
         suite.addTestSuite(SwapPathConstructionSelfTest.class);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java
index 7357d7e..99dd828 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java
@@ -37,6 +37,8 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridCa
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicNearRemoveFailureTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearRemoveFailureTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionDistributionTest;
+import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteChangingBaselineDownCacheRemoveFailoverTest;
+import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteChangingBaselineUpCacheRemoveFailoverTest;
 import org.apache.ignite.testframework.GridTestUtils;
 
 /**
@@ -74,6 +76,8 @@ public class IgniteCacheFailoverTestSuite extends TestSuite {
         suite.addTestSuite(GridCacheDhtClientRemoveFailureTest.class);
         suite.addTestSuite(GridCacheNearRemoveFailureTest.class);
         suite.addTestSuite(GridCacheAtomicNearRemoveFailureTest.class);
+        suite.addTestSuite(IgniteChangingBaselineUpCacheRemoveFailoverTest.class);
+        suite.addTestSuite(IgniteChangingBaselineDownCacheRemoveFailoverTest.class);
 
         suite.addTestSuite(IgniteCacheAtomicNodeJoinTest.class);
         suite.addTestSuite(IgniteCacheTxNodeJoinTest.class);

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite2.java
index 1150286..6668755 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite2.java
@@ -30,6 +30,8 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridCa
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedFailoverSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedTxSalvageSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedFailoverSelfTest;
+import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteChangingBaselineDownCachePutAllFailoverTest;
+import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteChangingBaselineUpCachePutAllFailoverTest;
 
 /**
  *
@@ -59,6 +61,8 @@ public class IgniteCacheFailoverTestSuite2 {
 
         suite.addTestSuite(CachePutAllFailoverAtomicTest.class);
         suite.addTestSuite(CachePutAllFailoverTxTest.class);
+        suite.addTestSuite(IgniteChangingBaselineDownCachePutAllFailoverTest.class);
+        suite.addTestSuite(IgniteChangingBaselineUpCachePutAllFailoverTest.class);
 
         return suite;
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index 3b071de..59425c8 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -28,7 +28,7 @@ import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePds
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsEvictionTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTreePageMemoryImplTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTreeReuseListPageMemoryImplTest;
-import org.apache.ignite.internal.processors.cache.persistence.pagemem.MetadataStoragePageMemoryImplTest;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.IndexStoragePageMemoryImplTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImplNoLoadTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImplTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PagesWriteThrottleSmokeTest;
@@ -36,6 +36,7 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentedRing
 import org.apache.ignite.internal.processors.database.IgniteDbClientNearCachePutGetTest;
 import org.apache.ignite.internal.processors.database.IgniteDbDynamicCacheSelfTest;
 import org.apache.ignite.internal.processors.database.IgniteDbMultiNodePutGetTest;
+import org.apache.ignite.internal.processors.database.IgniteDbPutGetWithCacheStoreTest;
 import org.apache.ignite.internal.processors.database.IgniteDbSingleNodePutGetTest;
 import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeTinyPutGetTest;
 
@@ -53,7 +54,7 @@ public class IgnitePdsTestSuite extends TestSuite {
 
         // Basic PageMemory tests.
         suite.addTestSuite(PageMemoryImplNoLoadTest.class);
-        suite.addTestSuite(MetadataStoragePageMemoryImplTest.class);
+        suite.addTestSuite(IndexStoragePageMemoryImplTest.class);
         suite.addTestSuite(IgnitePdsEvictionTest.class);
         suite.addTestSuite(PageMemoryImplTest.class);
 
@@ -75,6 +76,7 @@ public class IgnitePdsTestSuite extends TestSuite {
         suite.addTestSuite(IgnitePdsSingleNodePutGetPersistenceTest.class);
         suite.addTestSuite(IgnitePdsDynamicCacheTest.class);
         suite.addTestSuite(IgnitePdsClientNearCachePutGetTest.class);
+        suite.addTestSuite(IgniteDbPutGetWithCacheStoreTest.class);
 
         suite.addTestSuite(IgniteClusterActivateDeactivateTestWithPersistence.class);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 17833ac..8dc318a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -25,10 +25,14 @@ import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsExchange
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPageSizesTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreDataStructuresTest;
+import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteAllBaselineNodesOnlineFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOfflineBaselineNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOnlineNodeOutOfBaselineFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPageEvictionDuringPartitionClearTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsRebalancingOnNotStableTopologyTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsTransactionsHangTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsWholeClusterRestartTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.checkpoint.IgniteCheckpointDirtyPagesForLowLoadTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.filename.IgniteUidAsConsistentIdMigrationTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushBackgroundSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushDefaultSelfTest;
@@ -94,8 +98,15 @@ public class IgnitePdsTestSuite2 extends TestSuite {
 
         suite.addTestSuite(IgniteWalSerializerVersionTest.class);
 
+        // BaselineTopology tests
+        suite.addTestSuite(IgniteAllBaselineNodesOnlineFullApiSelfTest.class);
+        suite.addTestSuite(IgniteOfflineBaselineNodeFullApiSelfTest.class);
+        suite.addTestSuite(IgniteOnlineNodeOutOfBaselineFullApiSelfTest.class);
+
         suite.addTestSuite(WalCompactionTest.class);
 
+        suite.addTestSuite(IgniteCheckpointDirtyPagesForLowLoadTest.class);
+
         return suite;
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
index f016e39..6039ae3 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
@@ -19,6 +19,8 @@ package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTest;
+import org.apache.ignite.internal.processors.cache.distributed.CacheBaselineTopologyTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteBaselineAffinityTopologyActivationTest;
 import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateCacheTest;
 import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateDataStreamerTest;
 import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateDataStructureTest;
@@ -70,6 +72,9 @@ public class IgniteStandByClusterSuite extends TestSuite {
         suite.addTestSuite(IgniteChangeGlobalStateFailOverTest.class);
         suite.addTestSuite(IgniteChangeGlobalStateServiceTest.class);
 
+        suite.addTestSuite(CacheBaselineTopologyTest.class);
+        suite.addTestSuite(IgniteBaselineAffinityTopologyActivationTest.class);
+
         return suite;
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
index d1cf930..e524d04 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
@@ -40,6 +40,7 @@ import org.apache.ignite.spi.discovery.ClusterMetricsSnapshotSerializeSelfTest;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.thread.GridThreadPoolExecutorServiceSelfTest;
 import org.apache.ignite.thread.IgniteThreadPoolSizeTest;
+import org.apache.ignite.util.GridCommandHandlerTest;
 import org.apache.ignite.util.GridIntListSelfTest;
 import org.apache.ignite.util.GridLongListSelfTest;
 import org.apache.ignite.util.GridMessageCollectionTest;
@@ -111,6 +112,9 @@ public class IgniteUtilSelfTestSuite extends TestSuite {
         //dbx
         suite.addTestSuite(PageIdUtilsSelfTest.class);
 
+        // control.sh
+        suite.addTestSuite(GridCommandHandlerTest.class);
+
         return suite;
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
new file mode 100644
index 0000000..b119bfa
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.util;
+
+import java.io.File;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.ConnectorConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.commandline.CommandHandler;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
+import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_UNEXPECTED_ERROR;
+
+/**
+ * Command line handler test.
+ */
+public class GridCommandHandlerTest extends GridCommonAbstractTest {
+    /**
+     * @return Folder in work directory.
+     * @throws IgniteCheckedException If failed to resolve folder name.
+     */
+    protected File folder(String folder) throws IgniteCheckedException {
+        return U.resolveWorkDirectory(U.defaultWorkDirectory(), folder, false);
+    }
+
+    /**
+     * Delete all files created by database engine during test.
+     */
+    private void cleanupDbFiles() throws IgniteCheckedException {
+        deleteRecursively(folder("db"));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        cleanupDbFiles();
+
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanupDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setConnectorConfiguration(new ConnectorConfiguration());
+
+        DataStorageConfiguration memCfg = new DataStorageConfiguration().setDefaultDataRegionConfiguration(
+            new DataRegionConfiguration().setMaxSize(100 * 1024 * 1024));
+
+        cfg.setDataStorageConfiguration(memCfg);
+
+        DataStorageConfiguration dsCfg = cfg.getDataStorageConfiguration();
+        dsCfg.setWalMode(WALMode.LOG_ONLY);
+        dsCfg.getDefaultDataRegionConfiguration().setPersistenceEnabled(true);
+
+        return cfg;
+    }
+
+    /**
+     * Test activation works via control.sh
+     *
+     * @throws Exception If failed.
+     */
+    public void testActivate() throws Exception {
+        Ignite ignite = startGrids(1);
+
+        assertFalse(ignite.active());
+
+        CommandHandler cmd = new CommandHandler();
+
+        assertEquals(EXIT_CODE_OK, cmd.execute("--activate"));
+
+        assertTrue(ignite.active());
+    }
+
+    /**
+     * Test deactivation works via control.sh
+     *
+     * @throws Exception If failed.
+     */
+    public void testDeactivate() throws Exception {
+        Ignite ignite = startGrids(1);
+
+        assertFalse(ignite.active());
+
+        ignite.active(true);
+
+        assertTrue(ignite.active());
+
+        CommandHandler cmd = new CommandHandler();
+
+        assertEquals(EXIT_CODE_OK, cmd.execute("--deactivate"));
+
+        assertFalse(ignite.active());
+    }
+
+    /**
+     * Test cluster active state works via control.sh
+     *
+     * @throws Exception If failed.
+     */
+    public void testState() throws Exception {
+        Ignite ignite = startGrids(1);
+
+        assertFalse(ignite.active());
+
+        CommandHandler cmd = new CommandHandler();
+
+        assertEquals(EXIT_CODE_OK, cmd.execute("--state"));
+
+        ignite.active(true);
+
+        assertEquals(EXIT_CODE_OK, cmd.execute("--state"));
+    }
+
+    /**
+     * Test baseline collect works via control.sh
+     *
+     * @throws Exception If failed.
+     */
+    public void testBaselineCollect() throws Exception {
+        Ignite ignite = startGrids(1);
+
+        assertFalse(ignite.active());
+
+        ignite.active(true);
+
+        CommandHandler cmd = new CommandHandler();
+
+        cmd.execute("--baseline");
+
+        assertEquals(1, ignite.cluster().currentBaselineTopology().size());
+    }
+
+    /**
+     * @param ignites Ignites.
+     * @return Local node consistent ID.
+     */
+    private String consistentIds(Ignite... ignites) {
+        String res = "";
+
+        for (Ignite ignite : ignites) {
+            String consistentId = ignite.cluster().localNode().consistentId().toString();
+
+            if (!F.isEmpty(res))
+                res += ", ";
+
+            res += consistentId;
+        }
+
+        return res;
+    }
+
+    /**
+     * Test baseline add items works via control.sh
+     *
+     * @throws Exception If failed.
+     */
+    public void testBaselineAdd() throws Exception {
+        Ignite ignite = startGrids(1);
+
+        assertFalse(ignite.active());
+
+        ignite.active(true);
+
+        CommandHandler cmd = new CommandHandler();
+
+        Ignite other = startGrid(2);
+
+        assertEquals(EXIT_CODE_OK, cmd.execute("--baseline", "add", consistentIds(other)));
+        assertEquals(EXIT_CODE_OK, cmd.execute("--baseline", "add", consistentIds(other)));
+
+        assertEquals(2, ignite.cluster().currentBaselineTopology().size());
+    }
+
+    /**
+     * Test baseline remove works via control.sh
+     *
+     * @throws Exception If failed.
+     */
+    public void testBaselineRemove() throws Exception {
+        Ignite ignite = startGrids(1);
+        Ignite other = startGrid("nodeToStop");
+
+        assertFalse(ignite.active());
+
+        ignite.active(true);
+
+        String offlineNodeConsId = consistentIds(other);
+
+        stopGrid("nodeToStop");
+
+        CommandHandler cmd = new CommandHandler();
+
+        assertEquals(EXIT_CODE_OK, cmd.execute("--baseline"));
+        assertEquals(EXIT_CODE_OK, cmd.execute("--baseline", "remove", offlineNodeConsId));
+
+        assertEquals(1, ignite.cluster().currentBaselineTopology().size());
+    }
+
+    /**
+     * Test baseline set works via control.sh
+     *
+     * @throws Exception If failed.
+     */
+    public void testBaselineSet() throws Exception {
+        Ignite ignite = startGrids(1);
+
+        assertFalse(ignite.active());
+
+        ignite.active(true);
+
+        Ignite other = startGrid(2);
+
+        CommandHandler cmd = new CommandHandler();
+
+        assertEquals(EXIT_CODE_OK, cmd.execute("--baseline", "set", consistentIds(ignite, other)));
+
+        assertEquals(2, ignite.cluster().currentBaselineTopology().size());
+
+        assertEquals(EXIT_CODE_UNEXPECTED_ERROR, cmd.execute("--baseline", "set", "invalidConsistentId"));
+    }
+
+    /**
+     * Test baseline set by topology version works via control.sh
+     *
+     * @throws Exception If failed.
+     */
+    public void testBaselineVersion() throws Exception {
+        Ignite ignite = startGrids(1);
+
+        assertFalse(ignite.active());
+
+        ignite.active(true);
+
+        CommandHandler cmd = new CommandHandler();
+
+        startGrid(2);
+
+        assertEquals(EXIT_CODE_OK, cmd.execute("--baseline"));
+
+        assertEquals(EXIT_CODE_OK, cmd.execute("--baseline", "version", String.valueOf(ignite.cluster().topologyVersion())));
+
+        assertEquals(2, ignite.cluster().currentBaselineTopology().size());
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/dev-utils/pom.xml
----------------------------------------------------------------------
diff --git a/modules/dev-utils/pom.xml b/modules/dev-utils/pom.xml
index a782638..e57b17d 100644
--- a/modules/dev-utils/pom.xml
+++ b/modules/dev-utils/pom.xml
@@ -58,6 +58,32 @@
                     <skip>true</skip>
                 </configuration>
             </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4.1</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                    <archive>
+                        <manifest>
+                            <mainClass>org.apache.ignite.development.utils.IgniteWalConverter</mainClass>
+                        </manifest>
+                    </archive>
+
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
 </project>


[12/17] ignite git commit: IEP-4 Baseline topology for persistent caches (Phase 1) Contributed by: Dmitriy Govorukhin Dmitry Pavlov Eduard Shangareev Ily

Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java
new file mode 100644
index 0000000..7daef3c
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java
@@ -0,0 +1,420 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Metadata storage.
+ */
+public class IndexStorageImpl implements IndexStorage {
+    /** Max index name length (bytes num) */
+    public static final int MAX_IDX_NAME_LEN = 255;
+
+    /** Reserved size for index name. Needed for backward compatibility. */
+    public static final int RESERVED_IDX_NAME_LEN = 768;
+
+    /** Bytes in byte. */
+    private static final int BYTE_LEN = 1;
+
+    /** Page memory. */
+    private final PageMemory pageMem;
+
+    /** Index tree. */
+    private final MetaTree metaTree;
+
+    /** Meta page reuse tree. */
+    private final ReuseList reuseList;
+
+    /** Cache group ID. */
+    private final int grpId;
+
+    /** */
+    private final int allocPartId;
+
+    /** */
+    private final byte allocSpace;
+
+    /**
+     * @param pageMem Page memory.
+     * @param wal Write ahead log manager.
+     */
+    public IndexStorageImpl(
+        final PageMemory pageMem,
+        final IgniteWriteAheadLogManager wal,
+        final AtomicLong globalRmvId,
+        final int grpId,
+        final int allocPartId,
+        final byte allocSpace,
+        final ReuseList reuseList,
+        final long rootPageId,
+        final boolean initNew
+    ) {
+        try {
+            this.pageMem = pageMem;
+            this.grpId = grpId;
+            this.allocPartId = allocPartId;
+            this.allocSpace = allocSpace;
+            this.reuseList = reuseList;
+
+            metaTree = new MetaTree(grpId, allocPartId, allocSpace, pageMem, wal, globalRmvId, rootPageId,
+                reuseList, MetaStoreInnerIO.VERSIONS, MetaStoreLeafIO.VERSIONS, initNew);
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteException(e);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public RootPage getOrAllocateForTree(final String idxName) throws IgniteCheckedException {
+        final MetaTree tree = metaTree;
+
+        synchronized (this) {
+            byte[] idxNameBytes = idxName.getBytes(StandardCharsets.UTF_8);
+
+            if (idxNameBytes.length > MAX_IDX_NAME_LEN)
+                throw new IllegalArgumentException("Too long encoded indexName [maxAllowed=" + MAX_IDX_NAME_LEN +
+                    ", currentLength=" + idxNameBytes.length + ", name=" + idxName + "]");
+
+            final IndexItem row = tree.findOne(new IndexItem(idxNameBytes, 0));
+
+            if (row == null) {
+                long pageId = 0;
+
+                if (reuseList != null)
+                    pageId = reuseList.takeRecycledPage();
+
+                pageId = pageId == 0 ? pageMem.allocatePage(grpId, allocPartId, allocSpace) : pageId;
+
+                tree.put(new IndexItem(idxNameBytes, pageId));
+
+                return new RootPage(new FullPageId(pageId, grpId), true);
+            }
+            else {
+                final FullPageId pageId = new FullPageId(row.pageId, grpId);
+
+                return new RootPage(pageId, false);
+            }
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public RootPage dropRootPage(final String idxName)
+        throws IgniteCheckedException {
+        byte[] idxNameBytes = idxName.getBytes(StandardCharsets.UTF_8);
+
+        final IndexItem row = metaTree.remove(new IndexItem(idxNameBytes, 0));
+
+        if (row != null) {
+            if (reuseList == null)
+                pageMem.freePage(grpId, row.pageId);
+        }
+
+        return row != null ? new RootPage(new FullPageId(row.pageId, grpId), false) : null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void destroy() throws IgniteCheckedException {
+        metaTree.destroy();
+    }
+
+    /**
+     *
+     */
+    private static class MetaTree extends BPlusTree<IndexItem, IndexItem> {
+        /** */
+        private final int allocPartId;
+
+        /** */
+        private final byte allocSpace;
+
+        /**
+         * @param pageMem Page memory.
+         * @param metaPageId Meta page ID.
+         * @param reuseList Reuse list.
+         * @param innerIos Inner IOs.
+         * @param leafIos Leaf IOs.
+         * @throws IgniteCheckedException If failed.
+         */
+        private MetaTree(
+            final int cacheId,
+            final int allocPartId,
+            final byte allocSpace,
+            final PageMemory pageMem,
+            final IgniteWriteAheadLogManager wal,
+            final AtomicLong globalRmvId,
+            final long metaPageId,
+            final ReuseList reuseList,
+            final IOVersions<? extends BPlusInnerIO<IndexItem>> innerIos,
+            final IOVersions<? extends BPlusLeafIO<IndexItem>> leafIos,
+            final boolean initNew
+        ) throws IgniteCheckedException {
+            super(treeName("meta", "Meta"), cacheId, pageMem, wal, globalRmvId, metaPageId, reuseList, innerIos, leafIos);
+
+            this.allocPartId = allocPartId;
+            this.allocSpace = allocSpace;
+
+            initTree(initNew);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected long allocatePageNoReuse() throws IgniteCheckedException {
+            return pageMem.allocatePage(groupId(), allocPartId, allocSpace);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected int compare(final BPlusIO<IndexItem> io, final long pageAddr, final int idx,
+            final IndexItem row) throws IgniteCheckedException {
+            final int off = ((IndexIO)io).getOffset(pageAddr, idx);
+
+            int shift = 0;
+
+            // Compare index names.
+            final int len = PageUtils.getUnsignedByte(pageAddr, off + shift);
+
+            shift += BYTE_LEN;
+
+            for (int i = 0; i < len && i < row.idxName.length; i++) {
+                final int cmp = Byte.compare(PageUtils.getByte(pageAddr, off + i + shift), row.idxName[i]);
+
+                if (cmp != 0)
+                    return cmp;
+            }
+
+            return Integer.compare(len, row.idxName.length);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected IndexItem getRow(final BPlusIO<IndexItem> io, final long pageAddr,
+            final int idx, Object ignore) throws IgniteCheckedException {
+            return readRow(pageAddr, ((IndexIO)io).getOffset(pageAddr, idx));
+        }
+    }
+
+    /**
+     *
+     */
+    private static class IndexItem {
+        /** */
+        private byte[] idxName;
+
+        /** */
+        private long pageId;
+
+        /**
+         * @param idxName Index name.
+         * @param pageId Page ID.
+         */
+        private IndexItem(final byte[] idxName, final long pageId) {
+            this.idxName = idxName;
+            this.pageId = pageId;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "I [idxName=" + new String(idxName) + ", pageId=" + U.hexLong(pageId) + ']';
+        }
+    }
+
+    /**
+     * Store row to buffer.
+     *
+     * @param pageAddr Page address.
+     * @param off Offset in buf.
+     * @param row Row to store.
+     */
+    private static void storeRow(
+        final long pageAddr,
+        int off,
+        final IndexItem row
+    ) {
+        // Index name length.
+        PageUtils.putUnsignedByte(pageAddr, off, row.idxName.length);
+        off++;
+
+        // Index name.
+        PageUtils.putBytes(pageAddr, off, row.idxName);
+        off += row.idxName.length;
+
+        // Page ID.
+        PageUtils.putLong(pageAddr, off, row.pageId);
+    }
+
+    /**
+     * Copy row data.
+     *
+     * @param dstPageAddr Destination page address.
+     * @param dstOff Destination buf offset.
+     * @param srcPageAddr Source page address.
+     * @param srcOff Src buf offset.
+     */
+    private static void storeRow(
+        final long dstPageAddr,
+        int dstOff,
+        final long srcPageAddr,
+        int srcOff
+    ) {
+        // Index name length.
+        final int len = PageUtils.getUnsignedByte(srcPageAddr, srcOff);
+        srcOff++;
+
+        PageUtils.putUnsignedByte(dstPageAddr, dstOff, len);
+        dstOff++;
+
+        PageHandler.copyMemory(srcPageAddr, srcOff, dstPageAddr, dstOff, len);
+        srcOff += len;
+        dstOff += len;
+
+        // Page ID.
+        PageUtils.putLong(dstPageAddr, dstOff, PageUtils.getLong(srcPageAddr, srcOff));
+    }
+
+    /**
+     * Read row from buffer.
+     *
+     * @param pageAddr Page address.
+     * @param off Offset.
+     * @return Read row.
+     */
+    private static IndexItem readRow(final long pageAddr, int off) {
+        // Index name length.
+        final int len = PageUtils.getUnsignedByte(pageAddr, off) & 0xFF;
+        off++;
+
+        // Index name.
+        final byte[] idxName = PageUtils.getBytes(pageAddr, off, len);
+        off += len;
+
+        // Page ID.
+        final long pageId = PageUtils.getLong(pageAddr, off);
+
+        return new IndexItem(idxName, pageId);
+    }
+
+    /**
+     *
+     */
+    private interface IndexIO {
+        /**
+         * @param pageAddr Page address.
+         * @param idx Index.
+         * @return Offset in buffer according to {@code idx}.
+         */
+        int getOffset(long pageAddr, int idx);
+    }
+
+    /**
+     *
+     */
+    public static final class MetaStoreInnerIO extends BPlusInnerIO<IndexItem> implements IndexIO {
+        /** */
+        public static final IOVersions<MetaStoreInnerIO> VERSIONS = new IOVersions<>(
+            new MetaStoreInnerIO(1)
+        );
+
+        /**
+         * @param ver Version.
+         */
+        private MetaStoreInnerIO(final int ver) {
+            // name bytes and 1 byte for length, 8 bytes pageId
+            super(T_METASTORE_INNER, ver, false, RESERVED_IDX_NAME_LEN + 1 + 8);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void storeByOffset(long pageAddr, int off, IndexItem row) throws IgniteCheckedException {
+            storeRow(pageAddr, off, row);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void store(final long dstPageAddr, final int dstIdx, final BPlusIO<IndexItem> srcIo,
+            final long srcPageAddr,
+            final int srcIdx) throws IgniteCheckedException {
+            storeRow(dstPageAddr, offset(dstIdx), srcPageAddr, ((IndexIO)srcIo).getOffset(srcPageAddr, srcIdx));
+        }
+
+        /** {@inheritDoc} */
+        @Override public IndexItem getLookupRow(final BPlusTree<IndexItem, ?> tree, final long pageAddr,
+            final int idx) throws IgniteCheckedException {
+            return readRow(pageAddr, offset(idx));
+        }
+
+        /** {@inheritDoc} */
+        @Override public int getOffset(long pageAddr, final int idx) {
+            return offset(idx);
+        }
+    }
+
+    /**
+     *
+     */
+    public static final class MetaStoreLeafIO extends BPlusLeafIO<IndexItem> implements IndexIO {
+        /** */
+        public static final IOVersions<MetaStoreLeafIO> VERSIONS = new IOVersions<>(
+            new MetaStoreLeafIO(1)
+        );
+
+        /**
+         * @param ver Version.
+         */
+        private MetaStoreLeafIO(final int ver) {
+            // 4 byte cache ID, UTF-16 symbols and 1 byte for length, 8 bytes pageId
+            super(T_METASTORE_LEAF, ver, RESERVED_IDX_NAME_LEN + 1 + 8);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void storeByOffset(long buf, int off, IndexItem row) throws IgniteCheckedException {
+            storeRow(buf, off, row);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void store(final long dstPageAddr,
+            final int dstIdx,
+            final BPlusIO<IndexItem> srcIo,
+            final long srcPageAddr,
+            final int srcIdx) throws IgniteCheckedException {
+            storeRow(dstPageAddr, offset(dstIdx), srcPageAddr, ((IndexIO)srcIo).getOffset(srcPageAddr, srcIdx));
+        }
+
+        /** {@inheritDoc} */
+        @Override public IndexItem getLookupRow(final BPlusTree<IndexItem, ?> tree,
+            final long pageAddr,
+            final int idx) throws IgniteCheckedException {
+            return readRow(pageAddr, offset(idx));
+        }
+
+        /** {@inheritDoc} */
+        @Override public int getOffset(long pageAddr, final int idx) {
+            return offset(idx);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetaStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetaStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetaStore.java
deleted file mode 100644
index c09ce4e..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetaStore.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence;
-
-import org.apache.ignite.IgniteCheckedException;
-
-/**
- * Meta store.
- */
-public interface MetaStore {
-    /**
-     * Get or allocate initial page for an index.
-     *
-     * @param idxName Index name.
-     * @return {@link RootPage} that keeps pageId, allocated flag that shows whether the page
-     *      was newly allocated, and rootId that is counter which increments each time new page allocated.
-     * @throws IgniteCheckedException If failed.
-     */
-    public RootPage getOrAllocateForTree(String idxName) throws IgniteCheckedException;
-
-    /**
-     * Deallocate index page and remove from tree.
-     *
-     * @param idxName Index name.
-     * @return Root ID or -1 if no page was removed.
-     * @throws IgniteCheckedException  If failed.
-     */
-    public RootPage dropRootPage(String idxName) throws IgniteCheckedException;
-
-    /**
-     * Destroy this meta store.
-     *
-     * @throws IgniteCheckedException  If failed.
-     */
-    public void destroy() throws IgniteCheckedException;
-}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetadataStorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetadataStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetadataStorage.java
deleted file mode 100644
index e667807..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetadataStorage.java
+++ /dev/null
@@ -1,420 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence;
-
-import java.nio.charset.StandardCharsets;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
-import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
-import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- * Metadata storage.
- */
-public class MetadataStorage implements MetaStore {
-    /** Max index name length (bytes num) */
-    public static final int MAX_IDX_NAME_LEN = 255;
-
-    /** Reserved size for index name. Needed for backward compatibility. */
-    public static final int RESERVED_IDX_NAME_LEN = 768;
-
-    /** Bytes in byte. */
-    private static final int BYTE_LEN = 1;
-
-    /** Page memory. */
-    private final PageMemory pageMem;
-
-    /** Index tree. */
-    private final MetaTree metaTree;
-
-    /** Meta page reuse tree. */
-    private final ReuseList reuseList;
-
-    /** Cache group ID. */
-    private final int grpId;
-
-    /** */
-    private final int allocPartId;
-
-    /** */
-    private final byte allocSpace;
-
-    /**
-     * @param pageMem Page memory.
-     * @param wal Write ahead log manager.
-     */
-    public MetadataStorage(
-        final PageMemory pageMem,
-        final IgniteWriteAheadLogManager wal,
-        final AtomicLong globalRmvId,
-        final int grpId,
-        final int allocPartId,
-        final byte allocSpace,
-        final ReuseList reuseList,
-        final long rootPageId,
-        final boolean initNew
-    ) {
-        try {
-            this.pageMem = pageMem;
-            this.grpId = grpId;
-            this.allocPartId = allocPartId;
-            this.allocSpace = allocSpace;
-            this.reuseList = reuseList;
-
-            metaTree = new MetaTree(grpId, allocPartId, allocSpace, pageMem, wal, globalRmvId, rootPageId,
-                reuseList, MetaStoreInnerIO.VERSIONS, MetaStoreLeafIO.VERSIONS, initNew);
-        }
-        catch (IgniteCheckedException e) {
-            throw new IgniteException(e);
-        }
-    }
-
-    /** {@inheritDoc} */
-    @Override public RootPage getOrAllocateForTree(final String idxName) throws IgniteCheckedException {
-        final MetaTree tree = metaTree;
-
-        synchronized (this) {
-            byte[] idxNameBytes = idxName.getBytes(StandardCharsets.UTF_8);
-
-            if (idxNameBytes.length > MAX_IDX_NAME_LEN)
-                throw new IllegalArgumentException("Too long encoded indexName [maxAllowed=" + MAX_IDX_NAME_LEN +
-                    ", currentLength=" + idxNameBytes.length + ", name=" + idxName + "]");
-
-            final IndexItem row = tree.findOne(new IndexItem(idxNameBytes, 0));
-
-            if (row == null) {
-                long pageId = 0;
-
-                if (reuseList != null)
-                    pageId = reuseList.takeRecycledPage();
-
-                pageId = pageId == 0 ? pageMem.allocatePage(grpId, allocPartId, allocSpace) : pageId;
-
-                tree.put(new IndexItem(idxNameBytes, pageId));
-
-                return new RootPage(new FullPageId(pageId, grpId), true);
-            }
-            else {
-                final FullPageId pageId = new FullPageId(row.pageId, grpId);
-
-                return new RootPage(pageId, false);
-            }
-        }
-    }
-
-    /** {@inheritDoc} */
-    @Override public RootPage dropRootPage(final String idxName)
-        throws IgniteCheckedException {
-        byte[] idxNameBytes = idxName.getBytes(StandardCharsets.UTF_8);
-
-        final IndexItem row = metaTree.remove(new IndexItem(idxNameBytes, 0));
-
-        if (row != null) {
-            if (reuseList == null)
-                pageMem.freePage(grpId, row.pageId);
-        }
-
-        return row != null ? new RootPage(new FullPageId(row.pageId, grpId), false) : null;
-    }
-
-    /** {@inheritDoc} */
-    @Override public void destroy() throws IgniteCheckedException {
-        metaTree.destroy();
-    }
-
-    /**
-     *
-     */
-    private static class MetaTree extends BPlusTree<IndexItem, IndexItem> {
-        /** */
-        private final int allocPartId;
-
-        /** */
-        private final byte allocSpace;
-
-        /**
-         * @param pageMem Page memory.
-         * @param metaPageId Meta page ID.
-         * @param reuseList Reuse list.
-         * @param innerIos Inner IOs.
-         * @param leafIos Leaf IOs.
-         * @throws IgniteCheckedException If failed.
-         */
-        private MetaTree(
-            final int cacheId,
-            final int allocPartId,
-            final byte allocSpace,
-            final PageMemory pageMem,
-            final IgniteWriteAheadLogManager wal,
-            final AtomicLong globalRmvId,
-            final long metaPageId,
-            final ReuseList reuseList,
-            final IOVersions<? extends BPlusInnerIO<IndexItem>> innerIos,
-            final IOVersions<? extends BPlusLeafIO<IndexItem>> leafIos,
-            final boolean initNew
-        ) throws IgniteCheckedException {
-            super(treeName("meta", "Meta"), cacheId, pageMem, wal, globalRmvId, metaPageId, reuseList, innerIos, leafIos);
-
-            this.allocPartId = allocPartId;
-            this.allocSpace = allocSpace;
-
-            initTree(initNew);
-        }
-
-        /** {@inheritDoc} */
-        @Override protected long allocatePageNoReuse() throws IgniteCheckedException {
-            return pageMem.allocatePage(groupId(), allocPartId, allocSpace);
-        }
-
-        /** {@inheritDoc} */
-        @Override protected int compare(final BPlusIO<IndexItem> io, final long pageAddr, final int idx,
-            final IndexItem row) throws IgniteCheckedException {
-            final int off = ((IndexIO)io).getOffset(pageAddr, idx);
-
-            int shift = 0;
-
-            // Compare index names.
-            final int len = PageUtils.getUnsignedByte(pageAddr, off + shift);
-
-            shift += BYTE_LEN;
-
-            for (int i = 0; i < len && i < row.idxName.length; i++) {
-                final int cmp = Byte.compare(PageUtils.getByte(pageAddr, off + i + shift), row.idxName[i]);
-
-                if (cmp != 0)
-                    return cmp;
-            }
-
-            return Integer.compare(len, row.idxName.length);
-        }
-
-        /** {@inheritDoc} */
-        @Override protected IndexItem getRow(final BPlusIO<IndexItem> io, final long pageAddr,
-            final int idx, Object ignore) throws IgniteCheckedException {
-            return readRow(pageAddr, ((IndexIO)io).getOffset(pageAddr, idx));
-        }
-    }
-
-    /**
-     *
-     */
-    private static class IndexItem {
-        /** */
-        private byte[] idxName;
-
-        /** */
-        private long pageId;
-
-        /**
-         * @param idxName Index name.
-         * @param pageId Page ID.
-         */
-        private IndexItem(final byte[] idxName, final long pageId) {
-            this.idxName = idxName;
-            this.pageId = pageId;
-        }
-
-        /** {@inheritDoc} */
-        @Override public String toString() {
-            return "I [idxName=" + new String(idxName) + ", pageId=" + U.hexLong(pageId) + ']';
-        }
-    }
-
-    /**
-     * Store row to buffer.
-     *
-     * @param pageAddr Page address.
-     * @param off Offset in buf.
-     * @param row Row to store.
-     */
-    private static void storeRow(
-        final long pageAddr,
-        int off,
-        final IndexItem row
-    ) {
-        // Index name length.
-        PageUtils.putUnsignedByte(pageAddr, off, row.idxName.length);
-        off++;
-
-        // Index name.
-        PageUtils.putBytes(pageAddr, off, row.idxName);
-        off += row.idxName.length;
-
-        // Page ID.
-        PageUtils.putLong(pageAddr, off, row.pageId);
-    }
-
-    /**
-     * Copy row data.
-     *
-     * @param dstPageAddr Destination page address.
-     * @param dstOff Destination buf offset.
-     * @param srcPageAddr Source page address.
-     * @param srcOff Src buf offset.
-     */
-    private static void storeRow(
-        final long dstPageAddr,
-        int dstOff,
-        final long srcPageAddr,
-        int srcOff
-    ) {
-        // Index name length.
-        final int len = PageUtils.getUnsignedByte(srcPageAddr, srcOff);
-        srcOff++;
-
-        PageUtils.putUnsignedByte(dstPageAddr, dstOff, len);
-        dstOff++;
-
-        PageHandler.copyMemory(srcPageAddr, srcOff, dstPageAddr, dstOff, len);
-        srcOff += len;
-        dstOff += len;
-
-        // Page ID.
-        PageUtils.putLong(dstPageAddr, dstOff, PageUtils.getLong(srcPageAddr, srcOff));
-    }
-
-    /**
-     * Read row from buffer.
-     *
-     * @param pageAddr Page address.
-     * @param off Offset.
-     * @return Read row.
-     */
-    private static IndexItem readRow(final long pageAddr, int off) {
-        // Index name length.
-        final int len = PageUtils.getUnsignedByte(pageAddr, off) & 0xFF;
-        off++;
-
-        // Index name.
-        final byte[] idxName = PageUtils.getBytes(pageAddr, off, len);
-        off += len;
-
-        // Page ID.
-        final long pageId = PageUtils.getLong(pageAddr, off);
-
-        return new IndexItem(idxName, pageId);
-    }
-
-    /**
-     *
-     */
-    private interface IndexIO {
-        /**
-         * @param pageAddr Page address.
-         * @param idx Index.
-         * @return Offset in buffer according to {@code idx}.
-         */
-        int getOffset(long pageAddr, int idx);
-    }
-
-    /**
-     *
-     */
-    public static final class MetaStoreInnerIO extends BPlusInnerIO<IndexItem> implements IndexIO {
-        /** */
-        public static final IOVersions<MetaStoreInnerIO> VERSIONS = new IOVersions<>(
-            new MetaStoreInnerIO(1)
-        );
-
-        /**
-         * @param ver Version.
-         */
-        private MetaStoreInnerIO(final int ver) {
-            // name bytes and 1 byte for length, 8 bytes pageId
-            super(T_METASTORE_INNER, ver, false, RESERVED_IDX_NAME_LEN + 1 + 8);
-        }
-
-        /** {@inheritDoc} */
-        @Override public void storeByOffset(long pageAddr, int off, IndexItem row) throws IgniteCheckedException {
-            storeRow(pageAddr, off, row);
-        }
-
-        /** {@inheritDoc} */
-        @Override public void store(final long dstPageAddr, final int dstIdx, final BPlusIO<IndexItem> srcIo,
-            final long srcPageAddr,
-            final int srcIdx) throws IgniteCheckedException {
-            storeRow(dstPageAddr, offset(dstIdx), srcPageAddr, ((IndexIO)srcIo).getOffset(srcPageAddr, srcIdx));
-        }
-
-        /** {@inheritDoc} */
-        @Override public IndexItem getLookupRow(final BPlusTree<IndexItem, ?> tree, final long pageAddr,
-            final int idx) throws IgniteCheckedException {
-            return readRow(pageAddr, offset(idx));
-        }
-
-        /** {@inheritDoc} */
-        @Override public int getOffset(long pageAddr, final int idx) {
-            return offset(idx);
-        }
-    }
-
-    /**
-     *
-     */
-    public static final class MetaStoreLeafIO extends BPlusLeafIO<IndexItem> implements IndexIO {
-        /** */
-        public static final IOVersions<MetaStoreLeafIO> VERSIONS = new IOVersions<>(
-            new MetaStoreLeafIO(1)
-        );
-
-        /**
-         * @param ver Version.
-         */
-        private MetaStoreLeafIO(final int ver) {
-            // 4 byte cache ID, UTF-16 symbols and 1 byte for length, 8 bytes pageId
-            super(T_METASTORE_LEAF, ver, RESERVED_IDX_NAME_LEN + 1 + 8);
-        }
-
-        /** {@inheritDoc} */
-        @Override public void storeByOffset(long buf, int off, IndexItem row) throws IgniteCheckedException {
-            storeRow(buf, off, row);
-        }
-
-        /** {@inheritDoc} */
-        @Override public void store(final long dstPageAddr,
-            final int dstIdx,
-            final BPlusIO<IndexItem> srcIo,
-            final long srcPageAddr,
-            final int srcIdx) throws IgniteCheckedException {
-            storeRow(dstPageAddr, offset(dstIdx), srcPageAddr, ((IndexIO)srcIo).getOffset(srcPageAddr, srcIdx));
-        }
-
-        /** {@inheritDoc} */
-        @Override public IndexItem getLookupRow(final BPlusTree<IndexItem, ?> tree,
-            final long pageAddr,
-            final int idx) throws IgniteCheckedException {
-            return readRow(pageAddr, offset(idx));
-        }
-
-        /** {@inheritDoc} */
-        @Override public int getOffset(long pageAddr, final int idx) {
-            return offset(idx);
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RecoveryDebug.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RecoveryDebug.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RecoveryDebug.java
new file mode 100644
index 0000000..54017b4
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RecoveryDebug.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.TimeZone;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
+import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.Nullable;
+
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
+
+/**
+ *
+ */
+public class RecoveryDebug implements AutoCloseable {
+    /** */
+    private static final ThreadLocal<SimpleDateFormat> sdf = new ThreadLocal<SimpleDateFormat>() {
+        /** {@inheritDoc} */
+        @Override protected SimpleDateFormat initialValue() {
+            SimpleDateFormat f = new SimpleDateFormat("dd-MM-yyyy-HH-mm-ss-SSS");
+
+            f.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+            return f;
+        }
+    };
+
+    /** */
+    @Nullable private final IgniteLogger log;
+
+    /** */
+    @Nullable private FileChannel fc;
+
+    /**
+     * @param constId Consistent ID.
+     */
+    public RecoveryDebug(Object constId, long time) {
+        this(constId, time, null);
+    }
+
+    /**
+     * @param constId Consistent ID.
+     * @param log Logger.
+     */
+    public RecoveryDebug(Object constId,long time, @Nullable IgniteLogger log) {
+        this.log = log;
+
+        try {
+            String workDir = U.defaultWorkDirectory();
+
+            File tmpDir = new File(workDir, "tmp");
+
+            if (!tmpDir.exists())
+                if (!tmpDir.mkdir())
+                    return;
+
+            File f = new File(tmpDir, "recovery-" +
+                sdf.get().format(new Date(time)) + "-" + constId  +".log");
+
+            f.createNewFile();
+
+            fc = FileChannel.open(Paths.get(f.getPath()), EnumSet.of(CREATE, READ, WRITE));
+        }
+        catch (IgniteCheckedException | IOException e) {
+            U.error(log, "Fail create recovery debug file.", e);
+
+            fc = null;
+        }
+    }
+
+    /**
+     * @param rec TX record to append.
+     * @return {@code this} for convenience.
+     */
+    public RecoveryDebug append(TxRecord rec) {
+        GridCacheVersion txVer = rec.nearXidVersion();
+
+        return fc == null ? this : appendFile(
+            "Tx record " + rec.state() + " " + rec.nearXidVersion() + " timestamp " + rec.timestamp()
+        );
+    }
+
+    /**
+     * @param rec Data record to append.
+     * @param unwrapKeyValue unwrap key and value flag.
+     * @return {@code this} for convenience.
+     */
+    public RecoveryDebug append(DataRecord rec, boolean unwrapKeyValue) {
+        if (fc == null)
+            return this;
+
+        append("Data record\n");
+
+        for (DataEntry dataEntry :  rec.writeEntries())
+            append("\t" + dataEntry.op() + " " + dataEntry.nearXidVersion() +
+                (unwrapKeyValue ? " " + dataEntry.key() + " " + dataEntry.value() : "") + "\n"
+            );
+
+        return this;
+    }
+
+    /**
+     * @param st Statement to append.
+     * @return {@code this} for convenience.
+     */
+    public RecoveryDebug append(Object st) {
+        return fc == null ? this : appendFile(st);
+    }
+
+    /**
+     * @param st Statement to append.
+     * @return {@code this} for convenience.
+     */
+    private RecoveryDebug appendFile(Object st) {
+        try {
+            fc.write(ByteBuffer.wrap(st.toString().getBytes()));
+        }
+        catch (IOException e) {
+            U.error(null, "Fail write to recovery dump file.", e);
+        }
+
+        return this;
+    }
+
+    /**
+     * Closes this debug insrance.
+     */
+    @Override public void close() {
+        if (fc != null) {
+            try {
+                fc.force(true);
+
+                fc.close();
+            }
+            catch (IOException e) {
+                U.error(null, "Fail close recovery dump file.", e);
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/Storable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/Storable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/Storable.java
new file mode 100644
index 0000000..ae200df
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/Storable.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+/**
+ * Simple interface for data, store in some RowStore.
+ */
+public interface Storable {
+    /**
+     * @param link Link for this row.
+     */
+    public void link(long link);
+
+    /**
+     * @return Link for this row.
+     */
+    public long link();
+
+    /**
+     * @return Partition.
+     */
+    public int partition();
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileDownloader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileDownloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileDownloader.java
new file mode 100644
index 0000000..3c54a49
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileDownloader.java
@@ -0,0 +1,189 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.channels.FileChannel;
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.lang.IgniteInClosureX;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Part of direct node to node file downloading
+ */
+public class FileDownloader {
+    /** */
+    private final IgniteLogger log;
+
+    /** */
+    private static final int CHUNK_SIZE = 1024 * 1024;
+
+    /** */
+    private final Path path;
+
+    /** */
+    private final AtomicLong size = new AtomicLong(-1);
+
+    /** */
+    private ServerSocketChannel serverChannel;
+
+    /** */
+    private volatile GridFutureAdapter<?> finishFut;
+
+    /** */
+    private final GridFutureAdapter<?> initFut = new GridFutureAdapter<>();
+
+    /**
+     *
+     */
+    public FileDownloader(IgniteLogger log, Path path) {
+        this.log = log;
+        this.path = path;
+    }
+
+    /**
+     *
+     */
+    public InetSocketAddress start() throws IgniteCheckedException {
+        try {
+            ServerSocketChannel ch = ServerSocketChannel.open();
+
+            ch.bind(null);
+
+            serverChannel = ch;
+
+            return (InetSocketAddress)ch.getLocalAddress();
+        }
+        catch (Exception ex) {
+            throw new IgniteCheckedException(ex);
+        }
+    }
+
+    /**
+     *
+     */
+    public void download(GridFutureAdapter<?> fut){
+        this.finishFut = fut;
+
+        final ServerSocketChannel ch = serverChannel;
+
+        fut.listen(new IgniteInClosureX<IgniteInternalFuture<?>>() {
+            @Override public void applyx(IgniteInternalFuture<?> future) throws IgniteCheckedException {
+                try {
+
+                    if (log != null && log.isInfoEnabled())
+                        log.info("Server socket closed " + ch.getLocalAddress());
+
+                    ch.close();
+                }
+                catch (Exception ex) {
+                    U.error(log, "Fail close socket.", ex);
+
+                    throw new IgniteCheckedException(ex);
+                }
+            }
+        });
+
+        FileChannel writeChannel = null;
+        SocketChannel readChannel = null;
+
+        try {
+            File f = new File(path.toUri().getPath());
+
+            if (f.exists())
+                f.delete();
+
+            File cacheWorkDir = f.getParentFile();
+
+            if (!cacheWorkDir.exists())
+                cacheWorkDir.mkdir();
+
+            writeChannel = FileChannel.open(path, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
+
+            initFut.onDone();
+
+            readChannel = serverChannel.accept();
+
+            long pos = 0;
+
+            long size = this.size.get();
+
+            while (size == -1 || pos < size) {
+                pos += writeChannel.transferFrom(readChannel, pos, CHUNK_SIZE);
+
+                if (size == -1)
+                    size = this.size.get();
+            }
+        }
+        catch (IOException ex) {
+            initFut.onDone(ex);
+
+            fut.onDone(ex);
+        }
+        finally {
+            try {
+                if (writeChannel != null)
+                    writeChannel.close();
+            }
+            catch (IOException ex) {
+                throw new IgniteException("Could not close file: " + path);
+            }
+
+            try {
+                if (readChannel != null)
+                    readChannel.close();
+            }
+            catch (IOException ex) {
+                throw new IgniteException("Could not close socket");
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    public void download(long size, Throwable th) {
+        try {
+            initFut.get();
+
+            if (th != null)
+                finishFut.onDone(th);
+            else {
+                if (!this.size.compareAndSet(-1, size))
+                    finishFut.onDone(new IgniteException("Size mismatch: " + this.size.get() + " != " + size));
+                else
+                    finishFut.onDone();
+            }
+
+        }
+        catch (IgniteCheckedException e) {
+            finishFut.onDone(e);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index 47f1d4d..053ab2b 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -292,8 +292,10 @@ public class FilePageStore implements PageStore {
         lock.writeLock().lock();
 
         try {
+            // Since we always have a meta-page in the store, never revert allocated counter to a value smaller than
+            // header + page.
             if (inited)
-                allocated.set(fileIO.size());
+                allocated.set(Math.max(headerSize() + pageSize, fileIO.size()));
 
             recover = false;
         }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 66af0dd..d406df6 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -48,11 +48,13 @@ import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
 import org.apache.ignite.internal.processors.cache.StoredCacheData;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.marshaller.Marshaller;
 import org.apache.ignite.marshaller.jdk.JdkMarshaller;
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -196,8 +198,23 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
     }
 
     /** {@inheritDoc} */
+    @Override public void initializeForMetastorage()
+        throws IgniteCheckedException {
+        int grpId = MetaStorage.METASTORAGE_CACHE_ID;
+
+        if (!idxCacheStores.containsKey(grpId)) {
+            CacheStoreHolder holder = initDir(new File(storeWorkDir, "metastorage"), grpId, 1);
+
+            CacheStoreHolder old = idxCacheStores.put(grpId, holder);
+
+            assert old == null : "Non-null old store holder for metastorage";
+        }
+    }
+
+    /** {@inheritDoc} */
     @Override public void storeCacheData(StoredCacheData cacheData, boolean overwrite) throws IgniteCheckedException {
-        File cacheWorkDir = cacheWorkDirectory(cacheData.config());
+        File cacheWorkDir = cacheWorkDir(cacheData.config());
+
         File file;
 
         checkAndInitCacheWorkDir(cacheWorkDir);
@@ -321,18 +338,10 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
     }
 
     /**
-     * @param ccfg Cache configuration.
-     * @return Cache work directory.
+     *
      */
-    private File cacheWorkDirectory(CacheConfiguration ccfg) {
-        String dirName;
-
-        if (ccfg.getGroupName() != null)
-            dirName = CACHE_GRP_DIR_PREFIX + ccfg.getGroupName();
-        else
-            dirName = CACHE_DIR_PREFIX + ccfg.getName();
-
-        return new File(storeWorkDir, dirName);
+    public Path getPath(boolean isSharedGroup, String cacheOrGroupName, int partId) {
+        return getPartitionFile(cacheWorkDir(isSharedGroup, cacheOrGroupName), partId).toPath();
     }
 
     /**
@@ -344,25 +353,36 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
     private CacheStoreHolder initForCache(CacheGroupDescriptor grpDesc, CacheConfiguration ccfg) throws IgniteCheckedException {
         assert !grpDesc.sharedGroup() || ccfg.getGroupName() != null : ccfg.getName();
 
-        File cacheWorkDir = cacheWorkDirectory(ccfg);
+        File cacheWorkDir = cacheWorkDir(ccfg);
+
+        return initDir(cacheWorkDir, grpDesc.groupId(), grpDesc.config().getAffinity().partitions());
+    }
 
+    /**
+     * @param cacheWorkDir Work directory.
+     * @param grpId Group ID.
+     * @param partitions Number of partitions.
+     * @return Cache store holder.
+     * @throws IgniteCheckedException If failed.
+     */
+    private CacheStoreHolder initDir(File cacheWorkDir, int grpId, int partitions) throws IgniteCheckedException {
         boolean dirExisted = checkAndInitCacheWorkDir(cacheWorkDir);
 
         File idxFile = new File(cacheWorkDir, INDEX_FILE_NAME);
 
         if (dirExisted && !idxFile.exists())
-            grpsWithoutIdx.add(grpDesc.groupId());
+            grpsWithoutIdx.add(grpId);
 
         FileVersionCheckingFactory pageStoreFactory = new FileVersionCheckingFactory(
             dsCfg.getFileIOFactory(), igniteCfg.getDataStorageConfiguration());
 
         FilePageStore idxStore = pageStoreFactory.createPageStore(PageMemory.FLAG_IDX, idxFile);
 
-        FilePageStore[] partStores = new FilePageStore[grpDesc.config().getAffinity().partitions()];
+        FilePageStore[] partStores = new FilePageStore[partitions];
 
         for (int partId = 0; partId < partStores.length; partId++) {
             FilePageStore partStore = pageStoreFactory.createPageStore(
-                PageMemory.FLAG_DATA, new File(cacheWorkDir, String.format(PART_FILE_TEMPLATE, partId)));
+                PageMemory.FLAG_DATA, getPartitionFile(cacheWorkDir, partId));
 
             partStores[partId] = partStore;
         }
@@ -372,6 +392,14 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
 
     /**
      * @param cacheWorkDir Cache work directory.
+     * @param partId Partition id.
+     */
+    @NotNull private File getPartitionFile(File cacheWorkDir, int partId) {
+        return new File(cacheWorkDir, String.format(PART_FILE_TEMPLATE, partId));
+    }
+
+    /**
+     * @param cacheWorkDir Cache work directory.
      */
     private boolean checkAndInitCacheWorkDir(File cacheWorkDir) throws IgniteCheckedException {
         boolean dirExisted = false;
@@ -558,8 +586,21 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
      * @return Store dir for given cache.
      */
     public File cacheWorkDir(CacheConfiguration ccfg) {
-        String dirName = ccfg.getGroupName() == null ?
-            CACHE_DIR_PREFIX + ccfg.getName() : CACHE_GRP_DIR_PREFIX + ccfg.getGroupName();
+        boolean isSharedGrp = ccfg.getGroupName() != null;
+
+        return cacheWorkDir(isSharedGrp, isSharedGrp ? ccfg.getGroupName() : ccfg.getName());
+    }
+
+    /**
+     *
+     */
+    public File cacheWorkDir(boolean isSharedGroup, String cacheOrGroupName) {
+        String dirName;
+
+        if (isSharedGroup)
+            dirName = CACHE_GRP_DIR_PREFIX + cacheOrGroupName;
+        else
+            dirName = CACHE_DIR_PREFIX + cacheOrGroupName;
 
         return new File(storeWorkDir, dirName);
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileUploader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileUploader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileUploader.java
new file mode 100644
index 0000000..ba21ae9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileUploader.java
@@ -0,0 +1,100 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.SocketChannel;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+
+/**
+ * Part of direct node to node file downloading
+ */
+public class FileUploader {
+    /** */
+    private static final int CHUNK_SIZE = 1024 * 1024;
+
+    /** */
+    private final Path path;
+
+    /**
+     *
+     */
+    public FileUploader(Path path) {
+        this.path = path;
+    }
+
+    /**
+     *
+     */
+    public void upload(SocketChannel writeChannel, GridFutureAdapter<Long> finishFut) {
+        FileChannel readChannel = null;
+
+        try {
+            File file = new File(path.toUri().getPath());
+
+            if (!file.exists()) {
+                finishFut.onDone(
+                    new IgniteCheckedException(
+                        new FileNotFoundException(file.getAbsolutePath())
+                    )
+                );
+
+                return;
+            }
+
+            readChannel = FileChannel.open(path, StandardOpenOption.READ);
+
+            long written = 0;
+
+            long size = readChannel.size();
+
+            while (written < size)
+                written += readChannel.transferTo(written, CHUNK_SIZE, writeChannel);
+
+            finishFut.onDone(written);
+        }
+        catch (IOException ex) {
+            finishFut.onDone(ex);
+        }
+        finally {
+            //FIXME: when an error occurs on writeChannel.close() no attempt to close readChannel will happen. Need to be fixed.
+            try {
+                if (writeChannel != null)
+                    writeChannel.close();
+            }
+            catch (IOException ex) {
+                throw new IgniteException("Could not close socket.");
+            }
+
+            try {
+                if (readChannel != null)
+                    readChannel.close();
+            }
+            catch (IOException ex) {
+                throw new IgniteException("Could not close file: " + path);
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java
new file mode 100644
index 0000000..83ff91b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java
@@ -0,0 +1,141 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+/**
+ * {@link FileIO} that allows to work with ZIP compressed file.
+ * Doesn't allow random access and setting {@link FileIO#position()} backwards.
+ * Allows sequential reads including setting {@link FileIO#position()} forward.
+ */
+public class UnzipFileIO implements FileIO {
+    /** Zip input stream. */
+    private final ZipInputStream zis;
+
+    /** Byte array for draining data. */
+    private final byte[] arr = new byte[128 * 1024];
+
+    /** Size of uncompressed data. */
+    private final long size;
+
+    /** Total bytes read counter. */
+    private long totalBytesRead = 0;
+
+    /**
+     * @param zip Compressed file.
+     */
+    public UnzipFileIO(File zip) throws IOException {
+        zis = new ZipInputStream(new BufferedInputStream(new FileInputStream(zip)));
+
+        ZipEntry entry = zis.getNextEntry();
+        size = entry.getSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long position() throws IOException {
+        return totalBytesRead;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void position(long newPosition) throws IOException {
+        if (newPosition == totalBytesRead)
+            return;
+
+        if (newPosition < totalBytesRead)
+            throw new UnsupportedOperationException("Seeking backwards is not supported.");
+
+        long bytesRemaining = newPosition - totalBytesRead;
+
+        while (bytesRemaining > 0) {
+            int bytesToRead = bytesRemaining > arr.length ? arr.length : (int)bytesRemaining;
+
+            bytesRemaining -= zis.read(arr, 0, bytesToRead);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public int read(ByteBuffer dstBuf) throws IOException {
+        int bytesRead = zis.read(arr, 0, Math.min(dstBuf.remaining(), arr.length));
+
+        if (bytesRead == -1)
+            return -1;
+
+        dstBuf.put(arr, 0, bytesRead);
+
+        totalBytesRead += bytesRead;
+
+        return bytesRead;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int read(ByteBuffer dstBuf, long position) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int read(byte[] buf, int off, int len) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int write(ByteBuffer srcBuf) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int write(ByteBuffer srcBuf, long position) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void write(byte[] buf, int off, int len) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void force() throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long size() throws IOException {
+        return size;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clear() throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /** {@inheritDoc} */
+    @Override public MappedByteBuffer map(int maxWalSegmentSize) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void close() throws IOException {
+        zis.close();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsConsistentIdProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsConsistentIdProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsConsistentIdProcessor.java
index e7a7e63..ba6d822 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsConsistentIdProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsConsistentIdProcessor.java
@@ -161,6 +161,9 @@ public class PdsConsistentIdProcessor extends GridProcessorAdapter implements Pd
         if (!CU.isPersistenceEnabled(cfg))
             return compatibleResolve(pstStoreBasePath, consistentId);
 
+        if (ctx.clientNode())
+            return new PdsFolderSettings(pstStoreBasePath, UUID.randomUUID());
+
         if (getBoolean(IGNITE_DATA_STORAGE_FOLDER_BY_CONSISTENT_ID, false))
             return compatibleResolve(pstStoreBasePath, consistentId);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java
new file mode 100644
index 0000000..0847ca6
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java
@@ -0,0 +1,605 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.freelist;
+
+import java.util.concurrent.atomic.AtomicReferenceArray;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertFragmentRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageRemoveRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageUpdateRecord;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
+import org.apache.ignite.internal.processors.cache.persistence.Storable;
+import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictionTracker;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPagePayload;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseBag;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ */
+public abstract class AbstractFreeList<T extends Storable> extends PagesList implements FreeList<T>, ReuseList {
+    /** */
+    private static final int BUCKETS = 256; // Must be power of 2.
+
+    /** */
+    private static final int REUSE_BUCKET = BUCKETS - 1;
+
+    /** */
+    private static final Integer COMPLETE = Integer.MAX_VALUE;
+
+    /** */
+    private static final Integer FAIL_I = Integer.MIN_VALUE;
+
+    /** */
+    private static final Long FAIL_L = Long.MAX_VALUE;
+
+    /** */
+    private static final int MIN_PAGE_FREE_SPACE = 8;
+
+    /** */
+    private final int shift;
+
+    /** */
+    private final AtomicReferenceArray<Stripe[]> buckets = new AtomicReferenceArray<>(BUCKETS);
+
+    /** */
+    private final int MIN_SIZE_FOR_DATA_PAGE;
+
+    /** */
+    private final int emptyDataPagesBucket;
+
+    /** */
+    private final PageHandler<T, Boolean> updateRow = new UpdateRowHandler();
+
+    /** */
+    private final DataRegionMetricsImpl memMetrics;
+
+    /** */
+    private final PageEvictionTracker evictionTracker;
+
+    /**
+     *
+     */
+    private final class UpdateRowHandler extends PageHandler<T, Boolean> {
+        @Override public Boolean run(
+            int cacheId,
+            long pageId,
+            long page,
+            long pageAddr,
+            PageIO iox,
+            Boolean walPlc,
+            T row,
+            int itemId)
+            throws IgniteCheckedException {
+            AbstractDataPageIO<T> io = (AbstractDataPageIO<T>)iox;
+
+            int rowSize = io.getRowSize(row);
+
+            boolean updated = io.updateRow(pageAddr, itemId, pageSize(), null, row, rowSize);
+
+            evictionTracker.touchPage(pageId);
+
+            if (updated && needWalDeltaRecord(pageId, page, walPlc)) {
+                // TODO This record must contain only a reference to a logical WAL record with the actual data.
+                byte[] payload = new byte[rowSize];
+
+                DataPagePayload data = io.readPayload(pageAddr, itemId, pageSize());
+
+                assert data.payloadSize() == rowSize;
+
+                PageUtils.getBytes(pageAddr, data.offset(), payload, 0, rowSize);
+
+                wal.log(new DataPageUpdateRecord(
+                    cacheId,
+                    pageId,
+                    itemId,
+                    payload));
+            }
+
+            return updated;
+        }
+    }
+
+    /** */
+    private final PageHandler<T, Integer> writeRow = new WriteRowHandler();
+
+    /**
+     *
+     */
+    private final class WriteRowHandler extends PageHandler<T, Integer> {
+        @Override public Integer run(
+            int cacheId,
+            long pageId,
+            long page,
+            long pageAddr,
+            PageIO iox,
+            Boolean walPlc,
+            T row,
+            int written)
+            throws IgniteCheckedException {
+            AbstractDataPageIO<T> io = (AbstractDataPageIO<T>)iox;
+
+            int rowSize = io.getRowSize(row);
+            int oldFreeSpace = io.getFreeSpace(pageAddr);
+
+            assert oldFreeSpace > 0 : oldFreeSpace;
+
+            // If the full row does not fit into this page write only a fragment.
+            written = (written == 0 && oldFreeSpace >= rowSize) ? addRow(pageId, page, pageAddr, io, row, rowSize) :
+                addRowFragment(pageId, page, pageAddr, io, row, written, rowSize);
+
+            // Reread free space after update.
+            int newFreeSpace = io.getFreeSpace(pageAddr);
+
+            if (newFreeSpace > MIN_PAGE_FREE_SPACE) {
+                int bucket = bucket(newFreeSpace, false);
+
+                put(null, pageId, page, pageAddr, bucket);
+            }
+
+            if (written == rowSize)
+                evictionTracker.touchPage(pageId);
+
+            // Avoid boxing with garbage generation for usual case.
+            return written == rowSize ? COMPLETE : written;
+        }
+
+        /**
+         * @param pageId Page ID.
+         * @param page Page pointer.
+         * @param pageAddr Page address.
+         * @param io IO.
+         * @param row Row.
+         * @param rowSize Row size.
+         * @return Written size which is always equal to row size here.
+         * @throws IgniteCheckedException If failed.
+         */
+        private int addRow(
+            long pageId,
+            long page,
+            long pageAddr,
+            AbstractDataPageIO<T> io,
+            T row,
+            int rowSize
+        ) throws IgniteCheckedException {
+            io.addRow(pageId, pageAddr, row, rowSize, pageSize());
+
+            if (needWalDeltaRecord(pageId, page, null)) {
+                // TODO IGNITE-5829 This record must contain only a reference to a logical WAL record with the actual data.
+                byte[] payload = new byte[rowSize];
+
+                DataPagePayload data = io.readPayload(pageAddr, PageIdUtils.itemId(row.link()), pageSize());
+
+                assert data.payloadSize() == rowSize;
+
+                PageUtils.getBytes(pageAddr, data.offset(), payload, 0, rowSize);
+
+                wal.log(new DataPageInsertRecord(
+                    grpId,
+                    pageId,
+                    payload));
+            }
+
+            return rowSize;
+        }
+
+        /**
+         * @param pageId Page ID.
+         * @param page Page pointer.
+         * @param pageAddr Page address.
+         * @param io IO.
+         * @param row Row.
+         * @param written Written size.
+         * @param rowSize Row size.
+         * @return Updated written size.
+         * @throws IgniteCheckedException If failed.
+         */
+        private int addRowFragment(
+            long pageId,
+            long page,
+            long pageAddr,
+            AbstractDataPageIO<T> io,
+            T row,
+            int written,
+            int rowSize
+        ) throws IgniteCheckedException {
+            // Read last link before the fragment write, because it will be updated there.
+            long lastLink = row.link();
+
+            int payloadSize = io.addRowFragment(pageMem, pageAddr, row, written, rowSize, pageSize());
+
+            assert payloadSize > 0 : payloadSize;
+
+            if (needWalDeltaRecord(pageId, page, null)) {
+                // TODO IGNITE-5829 This record must contain only a reference to a logical WAL record with the actual data.
+                byte[] payload = new byte[payloadSize];
+
+                DataPagePayload data = io.readPayload(pageAddr, PageIdUtils.itemId(row.link()), pageSize());
+
+                PageUtils.getBytes(pageAddr, data.offset(), payload, 0, payloadSize);
+
+                wal.log(new DataPageInsertFragmentRecord(grpId, pageId, payload, lastLink));
+            }
+
+            return written + payloadSize;
+        }
+    }
+
+    /** */
+    private final PageHandler<Void, Long> rmvRow;
+
+    /**
+     *
+     */
+    private final class RemoveRowHandler extends PageHandler<Void, Long> {
+        /** Indicates whether partition ID should be masked from page ID. */
+        private final boolean maskPartId;
+
+        /** */
+        RemoveRowHandler(boolean maskPartId) {
+            this.maskPartId = maskPartId;
+        }
+
+        @Override public Long run(
+            int cacheId,
+            long pageId,
+            long page,
+            long pageAddr,
+            PageIO iox,
+            Boolean walPlc,
+            Void ignored,
+            int itemId)
+            throws IgniteCheckedException {
+            AbstractDataPageIO<T> io = (AbstractDataPageIO<T>)iox;
+
+            int oldFreeSpace = io.getFreeSpace(pageAddr);
+
+            assert oldFreeSpace >= 0 : oldFreeSpace;
+
+            long nextLink = io.removeRow(pageAddr, itemId, pageSize());
+
+            if (needWalDeltaRecord(pageId, page, walPlc))
+                wal.log(new DataPageRemoveRecord(cacheId, pageId, itemId));
+
+            int newFreeSpace = io.getFreeSpace(pageAddr);
+
+            if (newFreeSpace > MIN_PAGE_FREE_SPACE) {
+                int newBucket = bucket(newFreeSpace, false);
+
+                if (oldFreeSpace > MIN_PAGE_FREE_SPACE) {
+                    int oldBucket = bucket(oldFreeSpace, false);
+
+                    if (oldBucket != newBucket) {
+                        // It is possible that page was concurrently taken for put, in this case put will handle bucket change.
+                        pageId = maskPartId ? PageIdUtils.maskPartitionId(pageId) : pageId;
+                        if (removeDataPage(pageId, page, pageAddr, io, oldBucket))
+                            put(null, pageId, page, pageAddr, newBucket);
+                    }
+                }
+                else
+                    put(null, pageId, page, pageAddr, newBucket);
+
+                if (io.isEmpty(pageAddr))
+                    evictionTracker.forgetPage(pageId);
+            }
+
+            // For common case boxed 0L will be cached inside of Long, so no garbage will be produced.
+            return nextLink;
+        }
+    }
+
+    /**
+     * @param cacheId Cache ID.
+     * @param name Name (for debug purpose).
+     * @param memMetrics Memory metrics.
+     * @param memPlc Data region.
+     * @param reuseList Reuse list or {@code null} if this free list will be a reuse list for itself.
+     * @param wal Write ahead log manager.
+     * @param metaPageId Metadata page ID.
+     * @param initNew {@code True} if new metadata should be initialized.
+     * @throws IgniteCheckedException If failed.
+     */
+    public AbstractFreeList(
+        int cacheId,
+        String name,
+        DataRegionMetricsImpl memMetrics,
+        DataRegion memPlc,
+        ReuseList reuseList,
+        IgniteWriteAheadLogManager wal,
+        long metaPageId,
+        boolean initNew) throws IgniteCheckedException {
+        super(cacheId, name, memPlc.pageMemory(), BUCKETS, wal, metaPageId);
+
+        rmvRow = new RemoveRowHandler(cacheId == 0);
+
+        this.evictionTracker = memPlc.evictionTracker();
+        this.reuseList = reuseList == null ? this : reuseList;
+        int pageSize = pageMem.pageSize();
+
+        assert U.isPow2(pageSize) : "Page size must be a power of 2: " + pageSize;
+        assert U.isPow2(BUCKETS);
+        assert BUCKETS <= pageSize : pageSize;
+
+        // TODO this constant is used because currently we cannot reuse data pages as index pages
+        // TODO and vice-versa. It should be removed when data storage format is finalized.
+        MIN_SIZE_FOR_DATA_PAGE = pageSize - AbstractDataPageIO.MIN_DATA_PAGE_OVERHEAD;
+
+        int shift = 0;
+
+        while (pageSize > BUCKETS) {
+            shift++;
+            pageSize >>>= 1;
+        }
+
+        this.shift = shift;
+
+        this.memMetrics = memMetrics;
+
+        emptyDataPagesBucket = bucket(MIN_SIZE_FOR_DATA_PAGE, false);
+
+        init(metaPageId, initNew);
+    }
+
+    /**
+     * Calculates average fill factor over FreeListImpl instance.
+     *
+     * @return Tuple (numenator, denominator).
+     */
+    public T2<Long, Long> fillFactor() {
+        long pageSize = pageSize();
+
+        long totalSize = 0;
+        long loadSize = 0;
+
+        for (int b = BUCKETS - 2; b > 0; b--) {
+            long bsize = pageSize - ((REUSE_BUCKET - b) << shift);
+
+            long pages = bucketsSize[b].longValue();
+
+            loadSize += pages * (pageSize - bsize);
+
+            totalSize += pages * pageSize;
+        }
+
+        return totalSize == 0 ? new T2<>(0L, 0L) : new T2<>(loadSize, totalSize);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void dumpStatistics(IgniteLogger log) {
+        long dataPages = 0;
+
+        final boolean dumpBucketsInfo = false;
+
+        for (int b = 0; b < BUCKETS; b++) {
+            long size = bucketsSize[b].longValue();
+
+            if (!isReuseBucket(b))
+                dataPages += size;
+
+            if (dumpBucketsInfo) {
+                Stripe[] stripes = getBucket(b);
+
+                boolean empty = true;
+
+                if (stripes != null) {
+                    for (Stripe stripe : stripes) {
+                        if (!stripe.empty) {
+                            empty = false;
+
+                            break;
+                        }
+                    }
+                }
+
+                if (log.isInfoEnabled())
+                    log.info("Bucket [b=" + b +
+                        ", size=" + size +
+                        ", stripes=" + (stripes != null ? stripes.length : 0) +
+                        ", stripesEmpty=" + empty + ']');
+            }
+        }
+
+        if (dataPages > 0) {
+            if (log.isInfoEnabled())
+                log.info("FreeList [name=" + name +
+                    ", buckets=" + BUCKETS +
+                    ", dataPages=" + dataPages +
+                    ", reusePages=" + bucketsSize[REUSE_BUCKET].longValue() + "]");
+        }
+    }
+
+    /**
+     * @param freeSpace Page free space.
+     * @param allowReuse {@code True} if it is allowed to get reuse bucket.
+     * @return Bucket.
+     */
+    private int bucket(int freeSpace, boolean allowReuse) {
+        assert freeSpace > 0 : freeSpace;
+
+        int bucket = freeSpace >>> shift;
+
+        assert bucket >= 0 && bucket < BUCKETS : bucket;
+
+        if (!allowReuse && isReuseBucket(bucket))
+            bucket--;
+
+        return bucket;
+    }
+
+    /**
+     * @param part Partition.
+     * @return Page ID.
+     * @throws IgniteCheckedException If failed.
+     */
+    private long allocateDataPage(int part) throws IgniteCheckedException {
+        assert part <= PageIdAllocator.MAX_PARTITION_ID;
+        assert part != PageIdAllocator.INDEX_PARTITION;
+
+        return pageMem.allocatePage(grpId, part, PageIdAllocator.FLAG_DATA);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void insertDataRow(T row) throws IgniteCheckedException {
+        int rowSize = ioVersions().latest().getRowSize(row);
+
+        int written = 0;
+
+        do {
+            if (written != 0)
+                memMetrics.incrementLargeEntriesPages();
+
+            int freeSpace = Math.min(MIN_SIZE_FOR_DATA_PAGE, rowSize - written);
+
+            long pageId = 0L;
+
+            if (freeSpace == MIN_SIZE_FOR_DATA_PAGE)
+                pageId = takeEmptyPage(emptyDataPagesBucket, ioVersions());
+
+            boolean reuseBucket = false;
+
+            // TODO: properly handle reuse bucket.
+            if (pageId == 0L) {
+                for (int b = bucket(freeSpace, false) + 1; b < BUCKETS - 1; b++) {
+                    pageId = takeEmptyPage(b, ioVersions());
+
+                    if (pageId != 0L) {
+                        reuseBucket = isReuseBucket(b);
+
+                        break;
+                    }
+                }
+            }
+
+            boolean allocated = pageId == 0L;
+
+            if (allocated)
+                pageId = allocateDataPage(row.partition());
+            else
+                pageId = PageIdUtils.changePartitionId(pageId, (row.partition()));
+
+            AbstractDataPageIO<T> init = reuseBucket || allocated ? ioVersions().latest() : null;
+
+            written = write(pageId, writeRow, init, row, written, FAIL_I);
+
+            assert written != FAIL_I; // We can't fail here.
+        }
+        while (written != COMPLETE);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean updateDataRow(long link, T row) throws IgniteCheckedException {
+        assert link != 0;
+
+        long pageId = PageIdUtils.pageId(link);
+        int itemId = PageIdUtils.itemId(link);
+
+        Boolean updated = write(pageId, updateRow, row, itemId, null);
+
+        assert updated != null; // Can't fail here.
+
+        return updated;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeDataRowByLink(long link) throws IgniteCheckedException {
+        assert link != 0;
+
+        long pageId = PageIdUtils.pageId(link);
+        int itemId = PageIdUtils.itemId(link);
+
+        long nextLink = write(pageId, rmvRow, itemId, FAIL_L);
+
+        assert nextLink != FAIL_L; // Can't fail here.
+
+        while (nextLink != 0L) {
+            memMetrics.decrementLargeEntriesPages();
+
+            itemId = PageIdUtils.itemId(nextLink);
+            pageId = PageIdUtils.pageId(nextLink);
+
+            nextLink = write(pageId, rmvRow, itemId, FAIL_L);
+
+            assert nextLink != FAIL_L; // Can't fail here.
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected Stripe[] getBucket(int bucket) {
+        return buckets.get(bucket);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected boolean casBucket(int bucket, Stripe[] exp, Stripe[] upd) {
+        return buckets.compareAndSet(bucket, exp, upd);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected boolean isReuseBucket(int bucket) {
+        return bucket == REUSE_BUCKET;
+    }
+
+    /**
+     * @return Number of empty data pages in free list.
+     */
+    public int emptyDataPages() {
+        return bucketsSize[emptyDataPagesBucket].intValue();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void addForRecycle(ReuseBag bag) throws IgniteCheckedException {
+        assert reuseList == this : "not allowed to be a reuse list";
+
+        put(bag, 0, 0, 0L, REUSE_BUCKET);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long takeRecycledPage() throws IgniteCheckedException {
+        assert reuseList == this : "not allowed to be a reuse list";
+
+        return takeEmptyPage(REUSE_BUCKET, null);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long recycledPagesCount() throws IgniteCheckedException {
+        assert reuseList == this : "not allowed to be a reuse list";
+
+        return storedPagesCount(REUSE_BUCKET);
+    }
+
+    /**
+     * @return IOVersions.
+     */
+    public abstract IOVersions<? extends AbstractDataPageIO<T>> ioVersions();
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return "FreeList [name=" + name + ']';
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CacheFreeListImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CacheFreeListImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CacheFreeListImpl.java
new file mode 100644
index 0000000..dc0c92e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CacheFreeListImpl.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.freelist;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+
+/**
+ * FreeList implementation for cache.
+ */
+public class CacheFreeListImpl extends AbstractFreeList<CacheDataRow> {
+    /**
+     * @param cacheId Cache id.
+     * @param name Name.
+     * @param regionMetrics Region metrics.
+     * @param dataRegion Data region.
+     * @param reuseList Reuse list.
+     * @param wal Wal.
+     * @param metaPageId Meta page id.
+     * @param initNew Initialize new.
+     */
+    public CacheFreeListImpl(int cacheId, String name, DataRegionMetricsImpl regionMetrics, DataRegion dataRegion,
+        ReuseList reuseList,
+        IgniteWriteAheadLogManager wal, long metaPageId, boolean initNew) throws IgniteCheckedException {
+        super(cacheId, name, regionMetrics, dataRegion, reuseList, wal, metaPageId, initNew);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IOVersions<? extends AbstractDataPageIO<CacheDataRow>> ioVersions() {
+        return DataPageIO.VERSIONS;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return "FreeList [name=" + name + ']';
+    }
+}


[17/17] ignite git commit: IEP-4 Baseline topology for persistent caches (Phase 1) Contributed by: Dmitriy Govorukhin Dmitry Pavlov Eduard Shangareev Ily

Posted by ag...@apache.org.
IEP-4 Baseline topology for persistent caches (Phase 1)
Contributed by:
    Dmitriy Govorukhin <dm...@gmail.com>
    Dmitry Pavlov <dp...@gmail.com>
    Eduard Shangareev <ed...@gmail.com>
    Ilya Lantukh <il...@gridgain.com>
    Ivan Rakov <iv...@gmail.com>
    Sergey Chugunov <se...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6f7aba85
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6f7aba85
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6f7aba85

Branch: refs/heads/master
Commit: 6f7aba8526ff01a159b289aff932324a3604c1d8
Parents: 43a62cd
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Wed Jan 17 13:25:22 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Jan 17 13:30:12 2018 +0300

----------------------------------------------------------------------
 RELEASE_NOTES.txt                               |    8 +
 .../DummyPersistenceCompatibilityTest.java      |   25 +-
 .../testframework/util/MavenUtils.java          |   26 +-
 .../src/main/java/org/apache/ignite/Ignite.java |    4 +
 .../java/org/apache/ignite/IgniteCluster.java   |   39 +
 .../apache/ignite/IgniteSystemProperties.java   |   17 +
 .../org/apache/ignite/cluster/BaselineNode.java |   73 +
 .../org/apache/ignite/cluster/ClusterNode.java  |    8 +-
 .../configuration/AtomicConfiguration.java      |    2 +-
 .../configuration/DataRegionConfiguration.java  |    6 +
 .../configuration/DataStorageConfiguration.java |    8 +-
 .../configuration/IgniteConfiguration.java      |   37 +
 .../apache/ignite/internal/GridComponent.java   |    3 +
 .../ignite/internal/GridKernalContext.java      |    8 +-
 .../ignite/internal/GridKernalContextImpl.java  |   11 +
 .../internal/GridNodeOrderComparator.java       |   45 -
 .../ignite/internal/GridPluginComponent.java    |    5 +
 .../apache/ignite/internal/IgniteKernal.java    |   50 +-
 .../internal/cluster/DetachedClusterNode.java   |  121 ++
 .../cluster/IgniteClusterAsyncImpl.java         |   26 +
 .../internal/cluster/IgniteClusterImpl.java     |  172 +++
 .../internal/cluster/NodeOrderComparator.java   |   61 +
 .../cluster/NodeOrderLegacyComparator.java      |   46 +
 .../internal/commandline/CommandHandler.java    |  644 ++++++++-
 .../internal/managers/GridManagerAdapter.java   |   19 +
 .../managers/discovery/ConsistentIdMapper.java  |   96 +-
 .../internal/managers/discovery/DiscoCache.java |  105 +-
 .../discovery/GridDiscoveryManager.java         |  124 +-
 .../mem/unsafe/UnsafeMemoryProvider.java        |   12 +-
 .../pagemem/store/IgnitePageStoreManager.java   |    5 +
 .../pagemem/wal/IgniteWriteAheadLogManager.java |    5 +-
 .../wal/record/BaselineTopologyRecord.java      |   76 ++
 .../internal/pagemem/wal/record/CacheState.java |   24 +
 .../internal/pagemem/wal/record/DataEntry.java  |    4 +-
 .../internal/pagemem/wal/record/DataRecord.java |    6 +-
 .../pagemem/wal/record/ExchangeRecord.java      |   86 ++
 .../pagemem/wal/record/FilteredRecord.java      |    3 +
 .../pagemem/wal/record/MetastoreDataRecord.java |   62 +
 .../pagemem/wal/record/PageSnapshot.java        |    6 +-
 .../pagemem/wal/record/SnapshotRecord.java      |   12 +
 .../pagemem/wal/record/TimeStampRecord.java     |    8 +
 .../internal/pagemem/wal/record/TxRecord.java   |   50 +-
 .../internal/pagemem/wal/record/WALRecord.java  |   18 +-
 .../delta/DataPageInsertFragmentRecord.java     |    5 +-
 .../wal/record/delta/DataPageInsertRecord.java  |    5 +-
 .../wal/record/delta/DataPageRemoveRecord.java  |    5 +-
 .../delta/DataPageSetFreeListPageRecord.java    |    5 +-
 .../wal/record/delta/DataPageUpdateRecord.java  |    5 +-
 .../processors/GridProcessorAdapter.java        |    5 +
 .../affinity/GridAffinityAssignmentCache.java   |  100 +-
 .../cache/CacheAffinitySharedManager.java       |   90 +-
 .../cache/CacheJoinNodeDiscoveryData.java       |    2 +-
 .../processors/cache/ClusterCachesInfo.java     |   59 +-
 .../processors/cache/ExchangeActions.java       |   31 +-
 .../cache/ExchangeDiscoveryEvents.java          |   29 +-
 .../processors/cache/GridCacheAdapter.java      |   20 +
 .../processors/cache/GridCacheContext.java      |   70 +-
 .../processors/cache/GridCacheMapEntry.java     |   63 +-
 .../GridCachePartitionExchangeManager.java      |   63 +-
 .../processors/cache/GridCacheProcessor.java    |  172 ++-
 .../cache/GridCacheUpdateTxResult.java          |   21 +-
 .../processors/cache/GridCacheUtils.java        |   18 +-
 .../cache/IgniteCacheOffheapManagerImpl.java    |    7 +-
 .../cache/LocalJoinCachesContext.java           |  127 ++
 .../processors/cache/StateChangeRequest.java    |   42 +
 .../cache/binary/BinaryMetadataFileStore.java   |    4 +-
 .../binary/CacheObjectBinaryProcessorImpl.java  |  117 +-
 .../GridDistributedTxRemoteAdapter.java         |   17 +-
 .../dht/GridDhtAssignmentFetchFuture.java       |    4 +-
 .../distributed/dht/GridDhtCacheAdapter.java    |    4 +
 .../distributed/dht/GridDhtLocalPartition.java  |    3 +
 .../dht/GridDhtPartitionTopologyImpl.java       |   79 +-
 .../dht/GridDhtTopologyFutureAdapter.java       |    2 +-
 .../distributed/dht/GridDhtTxLocalAdapter.java  |    8 +-
 .../distributed/dht/GridDhtTxPrepareFuture.java |    5 +
 .../dht/GridPartitionedGetFuture.java           |   13 +-
 .../dht/GridPartitionedSingleGetFuture.java     |   18 +-
 .../atomic/GridNearAtomicFullUpdateRequest.java |    1 +
 .../colocated/GridDhtDetachedCacheEntry.java    |    8 +
 .../CachePartitionPartialCountersMap.java       |   17 +
 .../dht/preloader/GridDhtPartitionDemander.java |    2 +-
 .../GridDhtPartitionsExchangeFuture.java        |  231 +++-
 .../IgniteDhtPartitionsToReloadMap.java         |    7 +
 ...StopCachesOnClientReconnectExchangeTask.java |   59 +
 .../distributed/near/GridNearCacheEntry.java    |    7 +
 .../distributed/near/GridNearGetFuture.java     |   13 +-
 .../near/GridNearOptimisticTxPrepareFuture.java |   61 +-
 ...ridNearOptimisticTxPrepareFutureAdapter.java |    2 +-
 .../near/GridNearTxPrepareFutureAdapter.java    |    2 +-
 .../cache/persistence/CacheDataRow.java         |    2 +-
 .../persistence/DataRegionMetricsImpl.java      |    2 +-
 .../cache/persistence/DataStructure.java        |    3 +-
 .../GridCacheDatabaseSharedManager.java         |  668 +++++++++-
 .../persistence/GridCacheOffheapManager.java    |  416 +++---
 .../IgniteCacheDatabaseSharedManager.java       |   73 +-
 .../cache/persistence/IndexStorage.java         |   51 +
 .../cache/persistence/IndexStorageImpl.java     |  420 ++++++
 .../processors/cache/persistence/MetaStore.java |   51 -
 .../cache/persistence/MetadataStorage.java      |  420 ------
 .../cache/persistence/RecoveryDebug.java        |  170 +++
 .../processors/cache/persistence/Storable.java  |   38 +
 .../cache/persistence/file/FileDownloader.java  |  189 +++
 .../cache/persistence/file/FilePageStore.java   |    4 +-
 .../persistence/file/FilePageStoreManager.java  |   77 +-
 .../cache/persistence/file/FileUploader.java    |  100 ++
 .../cache/persistence/file/UnzipFileIO.java     |  141 ++
 .../filename/PdsConsistentIdProcessor.java      |    3 +
 .../persistence/freelist/AbstractFreeList.java  |  605 +++++++++
 .../persistence/freelist/CacheFreeListImpl.java |   59 +
 .../cache/persistence/freelist/FreeList.java    |    8 +-
 .../persistence/freelist/FreeListImpl.java      |  619 ---------
 .../cache/persistence/freelist/PagesList.java   |   11 +-
 .../persistence/metastorage/MetaStorage.java    |  450 +++++++
 .../metastorage/MetastorageDataRow.java         |   91 ++
 .../MetastorageLifecycleListener.java           |   47 +
 .../metastorage/MetastorageRowStore.java        |   96 ++
 .../metastorage/MetastorageSearchRow.java       |   38 +
 .../metastorage/MetastorageTree.java            |  266 ++++
 .../metastorage/MetsatorageSearchRowImpl.java   |   55 +
 .../metastorage/ReadOnlyMetastorage.java        |   28 +
 .../metastorage/ReadWriteMetastorage.java       |   32 +
 .../cache/persistence/pagemem/PageMemoryEx.java |   28 +-
 .../persistence/pagemem/PageMemoryImpl.java     |   57 +-
 .../partstate/PagesAllocationRange.java         |    7 +-
 .../partstate/PartitionAllocationMap.java       |   36 +-
 .../snapshot/IgniteCacheSnapshotManager.java    |   14 +
 .../snapshot/SnapshotDiscoveryMessage.java      |    7 +-
 .../cache/persistence/tree/BPlusTree.java       |    1 +
 .../persistence/tree/io/AbstractDataPageIO.java | 1256 ++++++++++++++++++
 .../cache/persistence/tree/io/DataPageIO.java   | 1214 +----------------
 .../cache/persistence/tree/io/PageIO.java       |   26 +-
 .../tree/io/PagePartitionCountersIO.java        |   34 +-
 .../tree/io/PagePartitionMetaIO.java            |   17 +-
 .../persistence/tree/io/SimpleDataPageIO.java   |  127 ++
 .../wal/AbstractWalRecordsIterator.java         |   20 +-
 .../wal/FileWriteAheadLogManager.java           |  230 +++-
 .../wal/reader/IgniteWalIteratorFactory.java    |   40 +-
 .../wal/reader/StandaloneGridKernalContext.java |    6 +
 .../reader/StandaloneWalRecordsIterator.java    |   65 +-
 .../BaselineTopologyRecordSerializer.java       |  168 +++
 .../wal/serializer/RecordDataV1Serializer.java  |   80 +-
 .../wal/serializer/RecordDataV2Serializer.java  |  224 +++-
 .../wal/serializer/RecordV1Serializer.java      |    6 +-
 .../wal/serializer/RecordV2Serializer.java      |   11 +-
 .../wal/serializer/TxRecordSerializer.java      |  152 +--
 .../store/GridCacheStoreManagerAdapter.java     |    6 +
 .../cache/transactions/IgniteTxAdapter.java     |   73 +-
 .../transactions/IgniteTxLocalAdapter.java      |   36 +-
 .../cache/transactions/IgniteTxManager.java     |    5 +
 .../CollectConflictPartitionKeysTask.java       |  235 ++++
 .../cache/verify/PartitionEntryHashRecord.java  |  237 ++++
 .../cache/verify/PartitionHashRecord.java       |  130 ++
 .../processors/cache/verify/PartitionKey.java   |  103 ++
 .../RetrieveConflictPartitionValuesTask.java    |  204 +++
 .../verify/VerifyBackupPartitionsTask.java      |  280 ++++
 .../processors/cluster/BaselineTopology.java    |  484 +++++++
 .../cluster/BaselineTopologyHistory.java        |  127 ++
 .../cluster/BaselineTopologyHistoryItem.java    |   79 ++
 .../processors/cluster/BranchingPointType.java  |   48 +
 .../cluster/ChangeGlobalStateFinishMessage.java |   19 +-
 .../cluster/ChangeGlobalStateMessage.java       |   38 +-
 .../cluster/DiscoveryDataClusterState.java      |   83 +-
 .../cluster/GridClusterStateProcessor.java      |  770 +++++++++--
 .../cluster/IGridClusterStateProcessor.java     |  122 ++
 .../AtomicDataStructureProxy.java               |    4 +-
 .../datastructures/DataStructuresProcessor.java |   50 +-
 .../processors/query/GridQueryProcessor.java    |    2 +-
 .../cluster/GridChangeStateCommandHandler.java  |    4 +-
 .../service/GridServiceProcessor.java           |    7 +-
 .../GridInternalSubscriptionProcessor.java      |   57 +
 .../util/lang/GridFilteredClosableIterator.java |   37 +
 .../util/lang/GridFilteredIterator.java         |    2 +-
 .../ignite/internal/util/lang/GridFunc.java     |   24 +
 .../visor/baseline/VisorBaselineNode.java       |   89 ++
 .../visor/baseline/VisorBaselineOperation.java  |   49 +
 .../visor/baseline/VisorBaselineTask.java       |  224 ++++
 .../visor/baseline/VisorBaselineTaskArg.java    |   99 ++
 .../visor/baseline/VisorBaselineTaskResult.java |  144 ++
 .../visor/compute/VisorGatewayTask.java         |    6 +-
 .../org/apache/ignite/spi/IgniteSpiAdapter.java |    6 +
 .../org/apache/ignite/spi/IgniteSpiContext.java |    7 +
 .../ignite/spi/discovery/tcp/ServerImpl.java    |   19 +-
 .../spi/discovery/tcp/TcpDiscoverySpi.java      |   20 +-
 .../tcp/internal/DiscoveryDataPacket.java       |   14 +-
 .../tcp/internal/TcpDiscoveryNode.java          |    7 +-
 .../resources/META-INF/classnames.properties    |   74 +-
 .../IgniteClientReconnectCacheTest.java         |    6 +-
 .../GridCacheAbstractRemoveFailureTest.java     |   76 +-
 .../cache/GridCacheAbstractSelfTest.java        |    3 +-
 .../processors/cache/GridCacheTestEntryEx.java  |    4 +-
 .../IgniteClusterActivateDeactivateTest.java    |   87 +-
 ...erActivateDeactivateTestWithPersistence.java |   75 +-
 .../distributed/CacheBaselineTopologyTest.java  |  711 ++++++++++
 .../distributed/CacheExchangeMergeTest.java     |   10 +-
 .../CacheLateAffinityAssignmentTest.java        |    5 +-
 .../CachePutAllFailoverAbstractTest.java        |   35 +-
 .../IgniteOptimisticTxSuspendResumeTest.java    |   19 +
 ...eBaselineAffinityTopologyActivationTest.java |  993 ++++++++++++++
 ...tePdsBinaryMetadataOnClusterRestartTest.java |  270 +++-
 .../IgnitePdsCacheRebalancingAbstractTest.java  |    7 +-
 ...IgnitePersistentStoreDataStructuresTest.java |    2 +
 ...teAllBaselineNodesOnlineFullApiSelfTest.java |   37 +
 .../IgniteBaselineAbstractFullApiSelfTest.java  |   50 +
 ...gingBaselineDownCachePutAllFailoverTest.java |  127 ++
 ...gingBaselineDownCacheRemoveFailoverTest.java |  157 +++
 ...angingBaselineUpCachePutAllFailoverTest.java |  122 ++
 ...angingBaselineUpCacheRemoveFailoverTest.java |  154 +++
 ...gniteOfflineBaselineNodeFullApiSelfTest.java |   41 +
 ...eOnlineNodeOutOfBaselineFullApiSelfTest.java |   39 +
 ...teStableBaselineCachePutAllFailoverTest.java |  104 ++
 ...teStableBaselineCacheRemoveFailoverTest.java |  130 ++
 ...gniteCheckpointDirtyPagesForLowLoadTest.java |  175 +++
 ...ckpointSimulationWithRealCpDisabledTest.java |   72 +-
 .../db/wal/IgniteWalFlushFailoverTest.java      |   21 +-
 ...lFlushMultiNodeFailoverAbstractSelfTest.java |    8 +-
 .../db/wal/IgniteWalRecoveryTest.java           |  284 +++-
 .../db/wal/IgniteWalSerializerVersionTest.java  |   38 +-
 .../persistence/db/wal/WalCompactionTest.java   |    8 +-
 .../db/wal/WalRecoveryTxLogicalRecordsTest.java |   11 +-
 .../db/wal/reader/IgniteWalReaderTest.java      |   17 +-
 .../persistence/file/FileDownloaderTest.java    |   90 ++
 .../pagemem/IndexStoragePageMemoryImplTest.java |  104 ++
 .../MetadataStoragePageMemoryImplTest.java      |  104 --
 .../pagemem/NoOpPageStoreManager.java           |    5 +
 .../persistence/pagemem/NoOpWALManager.java     |    2 +-
 .../IgniteChangeGlobalStateAbstractTest.java    |    1 +
 .../IgniteChangeGlobalStateFailOverTest.java    |    6 +-
 ...gniteAbstractStandByClientReconnectTest.java |    2 +
 ...eStandByClientReconnectToNewClusterTest.java |    6 +-
 .../database/CacheFreeListImplSelfTest.java     |  565 ++++++++
 .../database/FreeListImplSelfTest.java          |  565 --------
 .../IgniteDbPutGetWithCacheStoreTest.java       |  196 +++
 .../database/IndexStorageSelfTest.java          |  171 +++
 .../database/MetadataStorageSelfTest.java       |  171 ---
 .../testframework/GridSpiTestContext.java       |    6 +
 .../ignite/testframework/GridTestUtils.java     |    9 +
 .../junits/common/GridCommonAbstractTest.java   |   25 +-
 .../multijvm/IgniteClusterProcessProxy.java     |   26 +
 .../ignite/testsuites/IgniteBasicTestSuite.java |    8 +-
 .../IgniteCacheFailoverTestSuite.java           |    4 +
 .../IgniteCacheFailoverTestSuite2.java          |    4 +
 .../ignite/testsuites/IgnitePdsTestSuite.java   |    6 +-
 .../ignite/testsuites/IgnitePdsTestSuite2.java  |   11 +
 .../testsuites/IgniteStandByClusterSuite.java   |    5 +
 .../testsuites/IgniteUtilSelfTestSuite.java     |    6 +-
 .../ignite/util/GridCommandHandlerTest.java     |  274 ++++
 modules/dev-utils/pom.xml                       |   26 +
 .../development/utils/IgniteWalConverter.java   |   32 +-
 .../development/utils/RecordSizeCountStat.java  |   83 ++
 .../ignite/development/utils/TxWalStat.java     |  318 +++++
 .../ignite/development/utils/WalStat.java       |  327 +++++
 .../processors/query/h2/opt/GridH2Table.java    |    2 -
 .../IgniteBinaryObjectFieldsQuerySelfTest.java  |    7 +-
 ...eLockPartitionOnAffinityRunAbstractTest.java |    2 +-
 ...PartitionOnAffinityRunAtomicCacheOpTest.java |   18 +-
 .../IgniteCacheQueryNodeRestartSelfTest.java    |   56 +-
 ...ckPartitionOnAffinityRunAtomicCacheTest.java |   96 ++
 ...neLockPartitionOnAffinityRunTxCacheTest.java |   87 ++
 ...ngBaselineCacheQueryNodeRestartSelfTest.java |  125 ++
 ...StableBaselineBinObjFieldsQuerySelfTest.java |   78 ++
 ...eBaselineCacheQueryNodeRestartsSelfTest.java |   68 +
 .../IgniteCacheAffinityRunTestSuite.java        |    4 +
 .../IgniteCacheQuerySelfTestSuite.java          |    2 +
 .../IgniteCacheQuerySelfTestSuite2.java         |    4 +
 .../ApiParity/ClusterParityTest.cs              |    6 +-
 .../ApiParity/IgniteConfigurationParityTest.cs  |    3 +-
 266 files changed, 19765 insertions(+), 4726 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/RELEASE_NOTES.txt
----------------------------------------------------------------------
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index 27c5767..7af132b 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -1,6 +1,14 @@
 Apache Ignite Release Notes
 ===========================
 
+Apache Ignite In-Memory Data Fabric 2.4
+---------------------------------------
+Ignite:
+* Introduced Baseline Affinity Topology
+* Added ability to convert WAL to human-readable form
+* Added support for uninterruptible writes for WAL and storage
+* Local scan performance was improved up to 2 times in certain use-cases
+
 Apache Ignite In-Memory Data Fabric 2.3
 ---------------------------------------
 Ignite:

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/DummyPersistenceCompatibilityTest.java
----------------------------------------------------------------------
diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/DummyPersistenceCompatibilityTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/DummyPersistenceCompatibilityTest.java
index 64a1cbf..b34c921 100644
--- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/DummyPersistenceCompatibilityTest.java
+++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/DummyPersistenceCompatibilityTest.java
@@ -46,6 +46,9 @@ public class DummyPersistenceCompatibilityTest extends IgnitePersistenceCompatib
     /** */
     protected static final String TEST_CACHE_NAME = DummyPersistenceCompatibilityTest.class.getSimpleName();
 
+    /** */
+    protected volatile boolean compactFooter;
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         super.beforeTest();
@@ -59,11 +62,17 @@ public class DummyPersistenceCompatibilityTest extends IgnitePersistenceCompatib
 
         cfg.setPeerClassLoadingEnabled(false);
 
-        DataStorageConfiguration memCfg = new DataStorageConfiguration()
-            .setDefaultDataRegionConfiguration(
-                new DataRegionConfiguration().setPersistenceEnabled(true));
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                ));
 
-        cfg.setDataStorageConfiguration(memCfg);
+        cfg.setBinaryConfiguration(
+            new BinaryConfiguration()
+                .setCompactFooter(compactFooter)
+        );
 
         return cfg;
     }
@@ -102,7 +111,11 @@ public class DummyPersistenceCompatibilityTest extends IgnitePersistenceCompatib
      * @throws Exception If failed.
      */
     protected void doTestStartupWithOldVersion(String igniteVer, boolean compactFooter) throws Exception {
+        boolean prev = this.compactFooter;
+
         try {
+            this.compactFooter = compactFooter;
+
             startGrid(1, igniteVer, new ConfigurationClosure(compactFooter), new PostStartupClosure());
 
             stopAllGrids();
@@ -113,10 +126,12 @@ public class DummyPersistenceCompatibilityTest extends IgnitePersistenceCompatib
 
             ignite.active(true);
 
-            validateResultingCacheData(ignite.getOrCreateCache(TEST_CACHE_NAME));
+            validateResultingCacheData(ignite.cache(TEST_CACHE_NAME));
         }
         finally {
             stopAllGrids();
+
+            this.compactFooter = prev;
         }
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java
----------------------------------------------------------------------
diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java
index b2c798d..14250c9 100644
--- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java
+++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java
@@ -39,6 +39,12 @@ public class MavenUtils {
     /** Path to Maven local repository. For caching. */
     private static String locRepPath = null;
 
+    /** */
+    private static final String GG_MVN_REPO = "http://www.gridgainsystems.com/nexus/content/repositories/external";
+
+    /** Set this flag to true if running PDS compatibility tests locally. */
+    private static boolean useGgRepo;
+
     /**
      * Gets a path to an artifact with given version and groupId=org.apache.ignite and artifactId={@code artifactName}.
      * <br>
@@ -123,7 +129,7 @@ public class MavenUtils {
      * @throws Exception In case of an error.
      */
     private static String defineMavenLocalRepositoryPath() throws Exception {
-        String output = exec("mvn help:effective-settings");
+        String output = exec(buildMvnCommand() + " help:effective-settings");
 
         int endTagPos = output.indexOf("</localRepository>");
 
@@ -141,7 +147,8 @@ public class MavenUtils {
     private static void downloadArtifact(String artifact) throws Exception {
         X.println("Downloading artifact... Identifier: " + artifact);
 
-        exec("mvn dependency:get -Dartifact=" + artifact);
+        exec(buildMvnCommand() + " dependency:get -Dartifact=" + artifact +
+            (useGgRepo ? " -DremoteRepositories=" + GG_MVN_REPO : ""));
 
         X.println("Download is finished");
     }
@@ -193,4 +200,19 @@ public class MavenUtils {
             throw e;
         }
     }
+
+    /**
+     * @return Maven executable command.
+     */
+    private static String buildMvnCommand() {
+        String m2Home = System.getenv("M2_HOME");
+
+        if (m2Home == null)
+            m2Home = System.getProperty("M2_HOME");
+
+        if (m2Home == null)
+            return "mvn";
+
+        return m2Home + "/bin/mvn" ;
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/Ignite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/Ignite.java b/modules/core/src/main/java/org/apache/ignite/Ignite.java
index c8de43b..4e05463 100644
--- a/modules/core/src/main/java/org/apache/ignite/Ignite.java
+++ b/modules/core/src/main/java/org/apache/ignite/Ignite.java
@@ -659,7 +659,9 @@ public interface Ignite extends AutoCloseable {
      * Checks Ignite grid is active or not active.
      *
      * @return {@code True} if grid is active. {@code False} If grid is not active.
+     * @deprecated Use {@link IgniteCluster#active()} instead.
      */
+    @Deprecated
     public boolean active();
 
     /**
@@ -667,7 +669,9 @@ public interface Ignite extends AutoCloseable {
      *
      * @param active If {@code True} start activation process. If {@code False} start deactivation process.
      * @throws IgniteException If there is an already started transaction or lock in the same thread.
+     * @deprecated Use {@link IgniteCluster#active(boolean)} instead.
      */
+    @Deprecated
     public void active(boolean active);
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/IgniteCluster.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteCluster.java b/modules/core/src/main/java/org/apache/ignite/IgniteCluster.java
index cb2484f..b80de0c 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteCluster.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteCluster.java
@@ -22,6 +22,7 @@ import java.util.Collection;
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentMap;
+import org.apache.ignite.cluster.BaselineNode;
 import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.cluster.ClusterStartNodeResult;
@@ -427,6 +428,44 @@ public interface IgniteCluster extends ClusterGroup, IgniteAsyncSupport {
      */
     @Nullable public IgniteFuture<?> clientReconnectFuture();
 
+    /**
+     * Checks Ignite grid is active or not active.
+     *
+     * @return {@code True} if grid is active. {@code False} If grid is not active.
+     */
+    public boolean active();
+
+    /**
+     * Changes Ignite grid state to active or inactive.
+     *
+     * @param active If {@code True} start activation process. If {@code False} start deactivation process.
+     * @throws IgniteException If there is an already started transaction or lock in the same thread.
+     */
+    public void active(boolean active);
+
+    /**
+     * Gets current baseline topology. If baseline topology was not set, will return {@code null}.
+     *
+     * @return Collection of nodes included to the current baseline topology.
+     */
+    @Nullable public Collection<BaselineNode> currentBaselineTopology();
+
+    /**
+     * Sets baseline topology. The cluster must be activated for this method to be called.
+     *
+     * @param baselineTop A collection of nodes to be included to the baseline topology.
+     */
+    public void setBaselineTopology(Collection<? extends BaselineNode> baselineTop);
+
+    /**
+     * Sets baseline topology constructed from the cluster topology of the given version (the method succeeds
+     * only if the cluster topology has not changed). All client and daemon nodes will be filtered out of the
+     * resulting baseline.
+     *
+     * @param topVer Topology version to set.
+     */
+    public void setBaselineTopology(long topVer);
+
     /** {@inheritDoc} */
     @Deprecated
     @Override public IgniteCluster withAsync();

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 7ffc13d..1bbaacd 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -760,6 +760,13 @@ public final class IgniteSystemProperties {
     public static final String IGNITE_WAL_SERIALIZER_VERSION = "IGNITE_WAL_SERIALIZER_VERSION";
 
     /**
+     * If the property is set Ignite will use legacy node comparator (based on node order) inste
+     *
+     * Default value is {@code false}.
+     */
+    public static final String IGNITE_USE_LEGACY_NODE_COMPARATOR = "IGNITE_USE_LEGACY_NODE_COMPARATOR";
+
+    /**
      * Property that indicates should be mapped byte buffer used or not.
      * Possible values: {@code true} and {@code false}.
      */
@@ -773,6 +780,16 @@ public final class IgniteSystemProperties {
     public static final String IGNITE_DATA_STORAGE_FOLDER_BY_CONSISTENT_ID = "IGNITE_DATA_STORAGE_FOLDER_BY_CONSISTENT_ID";
 
     /**
+     * Default value is {@code false}.
+     */
+    public static final String IGNITE_WAL_DEBUG_LOG_ON_RECOVERY = "IGNITE_WAL_DEBUG_LOG_ON_RECOVERY";
+
+    /**
+     * Number of checkpoint history entries held in memory.
+     */
+    public static final String IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE = "IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE";
+
+    /**
      * If this property is set to {@code true} enable logging in {@link GridClient}.
      */
     public static final String IGNITE_GRID_CLIENT_LOG_ENABLED = "IGNITE_GRID_CLIENT_LOG_ENABLED";

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/cluster/BaselineNode.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/cluster/BaselineNode.java b/modules/core/src/main/java/org/apache/ignite/cluster/BaselineNode.java
new file mode 100644
index 0000000..9564aa0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/cluster/BaselineNode.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cluster;
+
+import java.util.Map;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Interface representing a single node from baseline topology.
+ */
+public interface BaselineNode {
+    /**
+     * Gets consistent globally unique node ID. This method returns consistent node ID which
+     * survives node restarts.
+     *
+     * @return Consistent globally unique node ID.
+     */
+    public Object consistentId();
+
+    /**
+     * Gets a node attribute. Attributes are assigned to nodes at startup
+     * via {@link IgniteConfiguration#getUserAttributes()} method.
+     * <p>
+     * The system adds the following attributes automatically:
+     * <ul>
+     * <li>{@code {@link System#getProperties()}} - All system properties.</li>
+     * <li>{@code {@link System#getenv(String)}} - All environment properties.</li>
+     * <li>All attributes defined in {@link org.apache.ignite.internal.IgniteNodeAttributes}</li>
+     * </ul>
+     * <p>
+     * Note that attributes cannot be changed at runtime.
+     *
+     * @param <T> Attribute Type.
+     * @param name Attribute name. <b>Note</b> that attribute names starting with
+     *      {@code org.apache.ignite} are reserved for internal use.
+     * @return Attribute value or {@code null}.
+     */
+    @Nullable
+    public <T> T attribute(String name);
+
+    /**
+     * Gets all node attributes. Attributes are assigned to nodes at startup
+     * via {@link IgniteConfiguration#getUserAttributes()} method.
+     * <p>
+     * The system adds the following attributes automatically:
+     * <ul>
+     * <li>{@code {@link System#getProperties()}} - All system properties.</li>
+     * <li>{@code {@link System#getenv(String)}} - All environment properties.</li>
+     * <li>All attributes defined in {@link org.apache.ignite.internal.IgniteNodeAttributes}</li>
+     * </ul>
+     * <p>
+     * Note that attributes cannot be changed at runtime.
+     *
+     * @return All node attributes.
+     */
+    public Map<String, Object> attributes();
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/cluster/ClusterNode.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/cluster/ClusterNode.java b/modules/core/src/main/java/org/apache/ignite/cluster/ClusterNode.java
index e122ff6..97b3165 100644
--- a/modules/core/src/main/java/org/apache/ignite/cluster/ClusterNode.java
+++ b/modules/core/src/main/java/org/apache/ignite/cluster/ClusterNode.java
@@ -101,7 +101,7 @@ import org.jetbrains.annotations.Nullable;
  * that comes with JDK as it also provides ability to view any node parameter
  * as a graph.
  */
-public interface ClusterNode {
+public interface ClusterNode extends BaselineNode {
     /**
      * Gets globally unique node ID. A new ID is generated every time a node restarts.
      *
@@ -115,7 +115,7 @@ public interface ClusterNode {
      *
      * @return Consistent globally unique node ID.
      */
-    public Object consistentId();
+    @Override public Object consistentId();
 
     /**
      * Gets a node attribute. Attributes are assigned to nodes at startup
@@ -135,7 +135,7 @@ public interface ClusterNode {
      *      {@code org.apache.ignite} are reserved for internal use.
      * @return Attribute value or {@code null}.
      */
-    @Nullable public <T> T attribute(String name);
+    @Override @Nullable public <T> T attribute(String name);
 
     /**
      * Gets metrics snapshot for this node. Note that node metrics are constantly updated
@@ -167,7 +167,7 @@ public interface ClusterNode {
      *
      * @return All node attributes.
      */
-    public Map<String, Object> attributes();
+    @Override public Map<String, Object> attributes();
 
     /**
      * Gets collection of addresses this node is known by.

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/configuration/AtomicConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/AtomicConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/AtomicConfiguration.java
index 24cc173..b017fdf 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/AtomicConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/AtomicConfiguration.java
@@ -29,7 +29,7 @@ import static org.apache.ignite.cache.CacheMode.PARTITIONED;
  */
 public class AtomicConfiguration {
     /** */
-    public static final int DFLT_BACKUPS = 0;
+    public static final int DFLT_BACKUPS = 1;
 
     /** */
     public static final CacheMode DFLT_CACHE_MODE = PARTITIONED;

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java
index 4ae87e3..2950e9c 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java
@@ -19,6 +19,7 @@ package org.apache.ignite.configuration;
 import java.io.Serializable;
 import org.apache.ignite.DataRegionMetrics;
 import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.mxbean.DataRegionMetricsMXBean;
 
 import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME;
@@ -431,4 +432,9 @@ public final class DataRegionConfiguration implements Serializable {
 
         return this;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(DataRegionConfiguration.class, this);
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
index 381c315..f1439d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
@@ -23,6 +23,7 @@ import org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOF
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
 import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
@@ -209,7 +210,7 @@ public class DataStorageConfiguration implements Serializable {
     private int walTlbSize = DFLT_TLB_SIZE;
 
     /** WAl buffer size. */
-    private int walBuffSize/* = DFLT_WAL_BUFF_SIZE*/;
+    private int walBuffSize;
 
     /** Wal flush frequency in milliseconds. */
     private long walFlushFreq = DFLT_WAL_FLUSH_FREQ;
@@ -902,4 +903,9 @@ public class DataStorageConfiguration implements Serializable {
 
         return this;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(DataStorageConfiguration.class, this);
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
index fc1fb6b..7dc4711 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
@@ -199,6 +199,9 @@ public class IgniteConfiguration {
     /** Default value for active on start flag. */
     public static final boolean DFLT_ACTIVE_ON_START = true;
 
+    /** Default value for auto-activation flag. */
+    public static final boolean DFLT_AUTO_ACTIVATION = true;
+
     /** Default failure detection timeout in millis. */
     @SuppressWarnings("UnnecessaryBoxing")
     public static final Long DFLT_FAILURE_DETECTION_TIMEOUT = new Long(10_000);
@@ -470,6 +473,9 @@ public class IgniteConfiguration {
     /** Active on start flag. */
     private boolean activeOnStart = DFLT_ACTIVE_ON_START;
 
+    /** Auto-activation flag. */
+    private boolean autoActivation = DFLT_AUTO_ACTIVATION;
+
     /** */
     private long longQryWarnTimeout = DFLT_LONG_QRY_WARN_TIMEOUT;
 
@@ -514,6 +520,7 @@ public class IgniteConfiguration {
         addrRslvr = cfg.getAddressResolver();
         allResolversPassReq = cfg.isAllSegmentationResolversPassRequired();
         atomicCfg = cfg.getAtomicConfiguration();
+        autoActivation = cfg.isAutoActivationEnabled();
         binaryCfg = cfg.getBinaryConfiguration();
         dsCfg = cfg.getDataStorageConfiguration();
         memCfg = cfg.getMemoryConfiguration();
@@ -2275,6 +2282,36 @@ public class IgniteConfiguration {
     }
 
     /**
+     * Get the flag indicating that cluster is enabled to activate automatically.
+     *
+     * If it is set to {@code true} and BaselineTopology is set as well than cluster activates automatically
+     * when all nodes from the BaselineTopology join the cluster.
+     *
+     * <p>
+     * Default value is {@link #DFLT_AUTO_ACTIVATION}.
+     * <p>
+     *
+     * @return Auto activation enabled flag value.
+     */
+    public boolean isAutoActivationEnabled() {
+        return autoActivation;
+    }
+
+    /**
+     * Sets flag indicating whether the cluster is enabled to activate automatically.
+     * This value should be the same on all nodes in the cluster.
+     *
+     * @param autoActivation Auto activation enabled flag value.
+     * @return {@code this} instance.
+     * @see #isAutoActivationEnabled()
+     */
+    public IgniteConfiguration setAutoActivationEnabled(boolean autoActivation) {
+        this.autoActivation = autoActivation;
+
+        return this;
+    }
+
+    /**
      * Gets flag indicating whether cache sanity check is enabled. If enabled, then Ignite
      * will perform the following checks and throw an exception if check fails:
      * <ul>

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/GridComponent.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridComponent.java b/modules/core/src/main/java/org/apache/ignite/internal/GridComponent.java
index 93ffe95..1e32e4f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridComponent.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridComponent.java
@@ -146,6 +146,9 @@ public interface GridComponent {
      */
     @Nullable public IgniteNodeValidationResult validateNode(ClusterNode node);
 
+    /** */
+    @Nullable public IgniteNodeValidationResult validateNode(ClusterNode node, DiscoveryDataBag.JoiningNodeDiscoveryData discoData);
+
     /**
      * Gets unique component type to distinguish components providing discovery data. Must return non-null value
      * if component implements any of methods {@link #collectJoiningNodeData(DiscoveryDataBag)}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index 210b401..ce12b61 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -42,8 +42,8 @@ import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor;
 import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
 import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
 import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
-import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
 import org.apache.ignite.internal.processors.hadoop.HadoopHelper;
+import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
 import org.apache.ignite.internal.processors.igfs.IgfsHelper;
 import org.apache.ignite.internal.processors.igfs.IgfsProcessorAdapter;
 import org.apache.ignite.internal.processors.job.GridJobProcessor;
@@ -62,6 +62,7 @@ import org.apache.ignite.internal.processors.security.GridSecurityProcessor;
 import org.apache.ignite.internal.processors.segmentation.GridSegmentationProcessor;
 import org.apache.ignite.internal.processors.service.GridServiceProcessor;
 import org.apache.ignite.internal.processors.session.GridTaskSessionProcessor;
+import org.apache.ignite.internal.processors.subscription.GridInternalSubscriptionProcessor;
 import org.apache.ignite.internal.processors.task.GridTaskProcessor;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
 import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions;
@@ -649,4 +650,9 @@ public interface GridKernalContext extends Iterable<GridComponent> {
      * @return PDS mode folder name resolver, also generates consistent ID in case new folder naming is used
      */
     public PdsFoldersResolver pdsFolderResolver();
+
+    /**
+     * @return subscription processor to manage internal-only (strict node-local) subscriptions between components.
+     */
+    public GridInternalSubscriptionProcessor internalSubscriptionProcessor();
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index 1f0292c..36c6231 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -79,6 +79,7 @@ import org.apache.ignite.internal.processors.security.GridSecurityProcessor;
 import org.apache.ignite.internal.processors.segmentation.GridSegmentationProcessor;
 import org.apache.ignite.internal.processors.service.GridServiceProcessor;
 import org.apache.ignite.internal.processors.session.GridTaskSessionProcessor;
+import org.apache.ignite.internal.processors.subscription.GridInternalSubscriptionProcessor;
 import org.apache.ignite.internal.processors.task.GridTaskProcessor;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
 import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions;
@@ -381,6 +382,9 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
     /** PDS mode folder name resolver, also generates consistent ID in case new folder naming is used */
     private PdsFoldersResolver pdsFolderRslvr;
 
+    /** */
+    private GridInternalSubscriptionProcessor internalSubscriptionProc;
+
     /**
      * No-arg constructor is required by externalization.
      */
@@ -585,6 +589,8 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
             mappingProc = (GridMarshallerMappingProcessor)comp;
         else if (comp instanceof PdsFoldersResolver)
             pdsFolderRslvr = (PdsFoldersResolver)comp;
+        else if (comp instanceof GridInternalSubscriptionProcessor)
+            internalSubscriptionProc = (GridInternalSubscriptionProcessor)comp;
         else if (!(comp instanceof DiscoveryNodeValidationProcessor
                 || comp instanceof PlatformPluginProcessor))
             assert (comp instanceof GridPluginComponent) : "Unknown manager class: " + comp.getClass();
@@ -1068,6 +1074,11 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
         return platformProc;
     }
 
+    /** {@inheritDoc} */
+    @Override public GridInternalSubscriptionProcessor internalSubscriptionProcessor() {
+        return internalSubscriptionProc;
+    }
+
     /**
      * @param disconnected Disconnected flag.
      */

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/GridNodeOrderComparator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridNodeOrderComparator.java b/modules/core/src/main/java/org/apache/ignite/internal/GridNodeOrderComparator.java
deleted file mode 100644
index 1c60e07..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridNodeOrderComparator.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal;
-
-import java.io.Serializable;
-import java.util.Comparator;
-import org.apache.ignite.cluster.ClusterNode;
-
-/**
- * Node order comparator.
- */
-public class GridNodeOrderComparator implements Comparator<ClusterNode>, Serializable {
-    /** */
-    private static final long serialVersionUID = 0L;
-
-    /** */
-    public static final Comparator<ClusterNode> INSTANCE = new GridNodeOrderComparator();
-
-    /**
-     * Private constructor. Don't create this class, use {@link #INSTANCE}.
-     */
-    private GridNodeOrderComparator() {
-
-    }
-
-    /** {@inheritDoc} */
-    @Override public int compare(ClusterNode n1, ClusterNode n2) {
-        return n1.order() < n2.order() ? -1 : n1.order() > n2.order() ? 1 : n1.id().compareTo(n2.id());
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/GridPluginComponent.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridPluginComponent.java b/modules/core/src/main/java/org/apache/ignite/internal/GridPluginComponent.java
index fd59d24..1e49e5c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridPluginComponent.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridPluginComponent.java
@@ -118,6 +118,11 @@ public class GridPluginComponent implements GridComponent {
     }
 
     /** {@inheritDoc} */
+    @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode node, JoiningNodeDiscoveryData discoData) {
+        return null;
+    }
+
+    /** {@inheritDoc} */
     @Override public void printMemoryStats() {
         // No-op.
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index c62d59c..3bd2a15 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -83,6 +83,7 @@ import org.apache.ignite.Ignition;
 import org.apache.ignite.MemoryMetrics;
 import org.apache.ignite.PersistenceMetrics;
 import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cluster.BaselineNode;
 import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.cluster.ClusterMetrics;
 import org.apache.ignite.cluster.ClusterNode;
@@ -125,6 +126,7 @@ import org.apache.ignite.internal.processors.cache.persistence.filename.PdsConsi
 import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
 import org.apache.ignite.internal.processors.closure.GridClosureProcessor;
 import org.apache.ignite.internal.processors.cluster.ClusterProcessor;
+import org.apache.ignite.internal.processors.cluster.IGridClusterStateProcessor;
 import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor;
 import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
 import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
@@ -152,6 +154,7 @@ import org.apache.ignite.internal.processors.security.GridSecurityProcessor;
 import org.apache.ignite.internal.processors.segmentation.GridSegmentationProcessor;
 import org.apache.ignite.internal.processors.service.GridServiceProcessor;
 import org.apache.ignite.internal.processors.session.GridTaskSessionProcessor;
+import org.apache.ignite.internal.processors.subscription.GridInternalSubscriptionProcessor;
 import org.apache.ignite.internal.processors.task.GridTaskProcessor;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
 import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions;
@@ -771,6 +774,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
         // Ack various information.
         ackAsciiLogo();
         ackConfigUrl();
+        ackConfiguration(cfg);
         ackDaemon();
         ackOsInfo();
         ackLanguageRuntime();
@@ -835,6 +839,10 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
 
             cfg.getMarshaller().setContext(ctx.marshallerContext());
 
+            GridInternalSubscriptionProcessor subscriptionProc = new GridInternalSubscriptionProcessor(ctx);
+
+            startProcessor(subscriptionProc);
+
             ClusterProcessor clusterProc = new ClusterProcessor(ctx);
 
             startProcessor(clusterProc);
@@ -916,7 +924,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
                 startProcessor(new  GridAffinityProcessor(ctx));
                 startProcessor(createComponent(GridSegmentationProcessor.class, ctx));
                 startProcessor(createComponent(IgniteCacheObjectProcessor.class, ctx));
-                startProcessor(new GridClusterStateProcessor(ctx));
+                startProcessor(createComponent(IGridClusterStateProcessor.class, ctx));
                 startProcessor(new GridCacheProcessor(ctx));
                 startProcessor(new GridQueryProcessor(ctx));
                 startProcessor(new ClientListenerProcessor(ctx));
@@ -1763,6 +1771,17 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
             log.info("Config URL: " + System.getProperty(IGNITE_CONFIG_URL, "n/a"));
     }
 
+
+    /**
+     * Acks configuration.
+     */
+    private void ackConfiguration(IgniteConfiguration cfg) {
+        assert log != null;
+
+        if (log.isInfoEnabled())
+            log.info(cfg.toString());
+    }
+
     /**
      * Acks Logger configuration.
      */
@@ -3300,7 +3319,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
         guard();
 
         try {
-            return context().state().publicApiActiveState();
+            return context().state().publicApiActiveState(true);
         }
         finally {
             unguard();
@@ -3309,17 +3328,19 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
 
     /** {@inheritDoc} */
     @Override public void active(boolean active) {
-        guard();
+        cluster().active(active);
+    }
 
-        try {
-            context().state().changeGlobalState(active).get();
-        }
-        catch (IgniteCheckedException e) {
-            throw U.convertException(e);
-        }
-        finally {
-            unguard();
-        }
+    /** */
+    private Collection<BaselineNode> baselineNodes() {
+        Collection<ClusterNode> srvNodes = cluster().forServers().nodes();
+
+        ArrayList baselineNodes = new ArrayList(srvNodes.size());
+
+        for (ClusterNode clN : srvNodes)
+            baselineNodes.add(clN);
+
+        return baselineNodes;
     }
 
     /** {@inheritDoc} */
@@ -3614,7 +3635,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
      * @throws IgniteException if cluster in inActive state
      */
     private void checkClusterState() throws IgniteException {
-        if (!ctx.state().publicApiActiveState()) {
+        if (!ctx.state().publicApiActiveState(true)) {
             throw new IgniteException("Can not perform the operation because the cluster is inactive. Note, that " +
                 "the cluster is considered inactive by default if Ignite Persistent Store is used to let all the nodes " +
                 "join the cluster. To activate the cluster call Ignite.active(true).");
@@ -3792,6 +3813,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
         if (cls.equals(DiscoveryNodeValidationProcessor.class))
             return (T)new OsDiscoveryNodeValidationProcessor(ctx);
 
+        if (cls.equals(IGridClusterStateProcessor.class))
+            return (T)new GridClusterStateProcessor(ctx);
+
         Class<T> implCls = null;
 
         try {

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/cluster/DetachedClusterNode.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/DetachedClusterNode.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/DetachedClusterNode.java
new file mode 100644
index 0000000..2c72bb0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/DetachedClusterNode.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.cluster;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.cluster.ClusterMetrics;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteProductVersion;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Representation of cluster node that isn't currently present in cluster.
+ */
+public class DetachedClusterNode implements ClusterNode {
+    /** */
+    @GridToStringExclude
+    private final UUID uuid = UUID.randomUUID();
+
+    /** Consistent ID. */
+    @GridToStringInclude
+    private final Object consistentId;
+
+    /** Node attributes. */
+    @GridToStringInclude
+    private final Map<String, Object> attributes;
+
+    /**
+     * @param consistentId Consistent ID.
+     * @param attributes Node attributes.
+     */
+    public DetachedClusterNode(Object consistentId, Map<String, Object> attributes) {
+        this.consistentId = consistentId;
+        this.attributes = attributes;
+    }
+
+    /** {@inheritDoc} */
+    @Override public UUID id() {
+        return uuid;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Object consistentId() {
+        return consistentId;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public <T> T attribute(String name) {
+        return (T)attributes.get(name);
+    }
+
+    /** {@inheritDoc} */
+    @Override public ClusterMetrics metrics() {
+        throw new UnsupportedOperationException("Operation is not supported on DetachedClusterNode");
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map<String, Object> attributes() {
+        return attributes;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<String> addresses() {
+        throw new UnsupportedOperationException("Operation is not supported on DetachedClusterNode");
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<String> hostNames() {
+        throw new UnsupportedOperationException("Operation is not supported on DetachedClusterNode");
+    }
+
+    /** {@inheritDoc} */
+    @Override public long order() {
+        throw new UnsupportedOperationException("Operation is not supported on DetachedClusterNode");
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteProductVersion version() {
+        throw new UnsupportedOperationException("Operation is not supported on DetachedClusterNode");
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isLocal() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isDaemon() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isClient() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(DetachedClusterNode.class, this);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterAsyncImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterAsyncImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterAsyncImpl.java
index f2c14eb..e238cb7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterAsyncImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterAsyncImpl.java
@@ -30,6 +30,7 @@ import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteCluster;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.BaselineNode;
 import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.cluster.ClusterMetrics;
 import org.apache.ignite.cluster.ClusterNode;
@@ -306,6 +307,31 @@ public class IgniteClusterAsyncImpl extends AsyncSupportAdapter<IgniteCluster>
     }
 
     /** {@inheritDoc} */
+    @Override public boolean active() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void active(boolean active) {
+
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Collection<BaselineNode> currentBaselineTopology() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void setBaselineTopology(Collection<? extends BaselineNode> baselineTop) {
+
+    }
+
+    /** {@inheritDoc} */
+    @Override public void setBaselineTopology(long topVer) {
+
+    }
+
+    /** {@inheritDoc} */
     @Nullable @Override public IgniteFuture<?> clientReconnectFuture() {
         return cluster.clientReconnectFuture();
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java
index 0c95dc2..392b43c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java
@@ -30,6 +30,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ConcurrentMap;
@@ -37,6 +38,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteCluster;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.BaselineNode;
 import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.cluster.ClusterGroupEmptyException;
 import org.apache.ignite.cluster.ClusterNode;
@@ -45,6 +47,7 @@ import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteComponentType;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.IgniteFutureImpl;
@@ -284,6 +287,175 @@ public class IgniteClusterImpl extends ClusterGroupAdapter implements IgniteClus
     }
 
     /** {@inheritDoc} */
+    @Override public boolean active() {
+        guard();
+
+        try {
+            return ctx.state().publicApiActiveState(true);
+        }
+        finally {
+            unguard();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void active(boolean active) {
+        guard();
+
+        try {
+            ctx.state().changeGlobalState(active, baselineNodes(), false).get();
+        }
+        catch (IgniteCheckedException e) {
+            throw U.convertException(e);
+        }
+        finally {
+            unguard();
+        }
+    }
+
+    /** */
+    private Collection<BaselineNode> baselineNodes() {
+        Collection<ClusterNode> srvNodes = ctx.cluster().get().forServers().nodes();
+
+        ArrayList baselineNodes = new ArrayList(srvNodes.size());
+
+        for (ClusterNode clN : srvNodes)
+            baselineNodes.add(clN);
+
+        return baselineNodes;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Collection<BaselineNode> currentBaselineTopology() {
+        guard();
+
+        try {
+            BaselineTopology blt = ctx.state().clusterState().baselineTopology();
+
+            return blt != null ? blt.currentBaseline() : null;
+        }
+        finally {
+            unguard();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void setBaselineTopology(Collection<? extends BaselineNode> baselineTop) {
+        guard();
+
+        try {
+            if (isInMemoryMode())
+                return;
+
+            validateBeforeBaselineChange(baselineTop);
+
+            ctx.state().changeGlobalState(true, baselineTop, true).get();
+        }
+        catch (IgniteCheckedException e) {
+            throw U.convertException(e);
+        }
+        finally {
+            unguard();
+        }
+    }
+
+    /** */
+    private boolean isInMemoryMode() {
+        return !CU.isPersistenceEnabled(cfg);
+    }
+
+    /**
+     * Executes validation checks of cluster state and BaselineTopology before changing BaselineTopology to new one.
+     */
+    private void validateBeforeBaselineChange(Collection<? extends BaselineNode> baselineTop) {
+        if (!ctx.state().clusterState().active())
+            throw new IgniteException("Changing BaselineTopology on inactive cluster is not allowed.");
+
+        if (baselineTop != null) {
+            if (baselineTop.isEmpty())
+                throw new IgniteException("BaselineTopology must contain at least one node.");
+
+            Collection<Object> onlineNodes = onlineBaselineNodesRequestedForRemoval(baselineTop);
+
+            if (onlineNodes != null) {
+                if (!onlineNodes.isEmpty())
+                    throw new IgniteException("Removing online nodes from BaselineTopology is not supported: " + onlineNodes);
+            }
+            else
+                //should never happen, actually: if cluster was activated, we expect it to have a BaselineTopology.
+                throw new IgniteException("Previous BaselineTopology was not found.");
+        }
+    }
+
+    /** */
+    @Nullable private Collection<Object> onlineBaselineNodesRequestedForRemoval(Collection<? extends BaselineNode> newBlt) {
+        BaselineTopology blt = ctx.state().clusterState().baselineTopology();
+        Set<Object> bltConsIds;
+
+        if (blt == null)
+            return null;
+        else
+            bltConsIds = blt.consistentIds();
+
+        ArrayList<Object> onlineNodesRequestedForRemoval = new ArrayList<>();
+
+        Collection<Object> aliveNodesConsIds = getConsistentIds(ctx.discovery().aliveServerNodes());
+
+        Collection<Object> newBltConsIds = getConsistentIds(newBlt);
+
+        for (Object oldBltConsId : bltConsIds) {
+            if (aliveNodesConsIds.contains(oldBltConsId)) {
+                if (!newBltConsIds.contains(oldBltConsId))
+                    onlineNodesRequestedForRemoval.add(oldBltConsId);
+            }
+        }
+
+        return onlineNodesRequestedForRemoval;
+    }
+
+    /** */
+    private Collection<Object> getConsistentIds(Collection<? extends BaselineNode> nodes) {
+        ArrayList<Object> res = new ArrayList<>(nodes.size());
+
+        for (BaselineNode n : nodes)
+            res.add(n.consistentId());
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void setBaselineTopology(long topVer) {
+        guard();
+
+        try {
+            if (isInMemoryMode())
+                return;
+
+            Collection<ClusterNode> top = topology(topVer);
+
+            if (top == null)
+                throw new IgniteException("Topology version does not exist: " + topVer);
+
+            Collection<BaselineNode> target = new ArrayList<>(top.size());
+
+            for (ClusterNode node : top) {
+                if (!node.isClient())
+                    target.add(node);
+            }
+
+            validateBeforeBaselineChange(target);
+
+            ctx.state().changeGlobalState(true, target, true).get();
+        }
+        catch (IgniteCheckedException e) {
+            throw U.convertException(e);
+        }
+        finally {
+            unguard();
+        }
+    }
+
+    /** {@inheritDoc} */
     @Override public void enableStatistics(Collection<String> caches, boolean enabled) {
         guard();
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/cluster/NodeOrderComparator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/NodeOrderComparator.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/NodeOrderComparator.java
new file mode 100644
index 0000000..a493850
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/NodeOrderComparator.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.cluster;
+
+import java.io.Serializable;
+import java.util.Comparator;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cluster.ClusterNode;
+
+/**
+ * Node order comparator.
+ */
+public class NodeOrderComparator implements Comparator<ClusterNode>, Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private static final Comparator<ClusterNode> INSTANCE = new NodeOrderComparator();
+
+    /**
+     * @return Node comparator.
+     */
+    public static Comparator<ClusterNode> getInstance() {
+        return IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_USE_LEGACY_NODE_COMPARATOR) ?
+            NodeOrderLegacyComparator.INSTANCE : INSTANCE;
+    }
+
+    /**
+     * Private constructor. Don't create this class, use {@link #getInstance()}.
+     */
+    private NodeOrderComparator() {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public int compare(ClusterNode n1, ClusterNode n2) {
+        Object id1 = n1.consistentId();
+        Object id2 = n2.consistentId();
+
+        if (id1 instanceof Comparable && id2 instanceof Comparable && id1.getClass().equals(id2.getClass()))
+            return ((Comparable)id1).compareTo(id2);
+
+        return id1.toString().compareTo(id2.toString());
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/cluster/NodeOrderLegacyComparator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/NodeOrderLegacyComparator.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/NodeOrderLegacyComparator.java
new file mode 100644
index 0000000..18a6ac3
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/NodeOrderLegacyComparator.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.cluster;
+
+import java.io.Serializable;
+import java.util.Comparator;
+import org.apache.ignite.cluster.ClusterNode;
+
+/**
+ * Node order comparator.
+ */
+public class NodeOrderLegacyComparator implements Comparator<ClusterNode>, Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    public static final Comparator<ClusterNode> INSTANCE = new NodeOrderLegacyComparator();
+
+    /**
+     * Private constructor. Don't create this class, use {@link #INSTANCE}.
+     */
+    private NodeOrderLegacyComparator() {
+
+    }
+
+    /** {@inheritDoc} */
+    @Override public int compare(ClusterNode n1, ClusterNode n2) {
+        return n1.order() < n2.order() ? -1 : n1.order() > n2.order() ? 1 : n1.id().compareTo(n2.id());
+    }
+}
\ No newline at end of file


[15/17] ignite git commit: IEP-4 Baseline topology for persistent caches (Phase 1) Contributed by: Dmitriy Govorukhin Dmitry Pavlov Eduard Shangareev Ily

Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java
index 8ea2981..ed469a4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java
@@ -19,7 +19,8 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
 
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
@@ -68,7 +69,7 @@ public class DataPageUpdateRecord extends PageDeltaRecord {
     @Override public void applyDelta(PageMemory pageMem, long pageAddr) throws IgniteCheckedException {
         assert payload != null;
 
-        DataPageIO io = DataPageIO.VERSIONS.forPage(pageAddr);
+        AbstractDataPageIO io = PageIO.getPageIO(pageAddr);
 
         io.updateRow(pageAddr, itemId, pageMem.pageSize(), payload, null, 0);
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/GridProcessorAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/GridProcessorAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/GridProcessorAdapter.java
index d6f78ab..cda6850 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/GridProcessorAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/GridProcessorAdapter.java
@@ -152,6 +152,11 @@ public abstract class GridProcessorAdapter implements GridProcessor {
     }
 
     /** {@inheritDoc} */
+    @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode node, DiscoveryDataBag.JoiningNodeDiscoveryData discoData) {
+        return null;
+    }
+
+    /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(GridProcessorAdapter.class, this);
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
index c451b77..dd46246 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.affinity;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -38,9 +39,10 @@ import org.apache.ignite.cache.affinity.AffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.GridNodeOrderComparator;
+import org.apache.ignite.internal.cluster.NodeOrderComparator;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -48,7 +50,6 @@ import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.jetbrains.annotations.Nullable;
-import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_AFFINITY_HISTORY_SIZE;
 import static org.apache.ignite.IgniteSystemProperties.getInteger;
@@ -85,6 +86,12 @@ public class GridAffinityAssignmentCache {
     /** */
     private List<List<ClusterNode>> idealAssignment;
 
+    /** */
+    private BaselineTopology baselineTopology;
+
+    /** */
+    private List<List<ClusterNode>> baselineAssignment;
+
     /** Cache item corresponding to the head topology version. */
     private final AtomicReference<GridAffinityAssignment> head;
 
@@ -260,8 +267,8 @@ public class GridAffinityAssignmentCache {
      * @return Affinity assignments.
      */
     @SuppressWarnings("IfMayBeConditional")
-    public List<List<ClusterNode>> calculate(AffinityTopologyVersion topVer, DiscoveryEvent discoEvt,
-        DiscoCache discoCache) {
+    public List<List<ClusterNode>> calculate(AffinityTopologyVersion topVer, @Nullable DiscoveryEvent discoEvt,
+        @Nullable DiscoCache discoCache) {
         if (log.isDebugEnabled())
             log.debug("Calculating affinity [topVer=" + topVer + ", locNodeId=" + ctx.localNodeId() +
                 ", discoEvt=" + discoEvt + ']');
@@ -274,11 +281,20 @@ public class GridAffinityAssignmentCache {
         if (!locCache) {
             sorted = new ArrayList<>(discoCache.cacheGroupAffinityNodes(groupId()));
 
-            Collections.sort(sorted, GridNodeOrderComparator.INSTANCE);
+            Collections.sort(sorted, NodeOrderComparator.getInstance());
         }
         else
             sorted = Collections.singletonList(ctx.discovery().localNode());
 
+        boolean hasBaseline = false;
+        boolean changedBaseline = false;
+
+        if (discoCache != null) {
+            hasBaseline = discoCache.state().baselineTopology() != null;
+            changedBaseline = !hasBaseline ? baselineTopology != null :
+                !discoCache.state().baselineTopology().equals(baselineTopology);
+        }
+
         List<List<ClusterNode>> assignment;
 
         if (prevAssignment != null && discoEvt != null) {
@@ -286,18 +302,51 @@ public class GridAffinityAssignmentCache {
 
             if (!affNode)
                 assignment = prevAssignment;
+            else if (hasBaseline && !changedBaseline) {
+                if (baselineAssignment == null)
+                    baselineAssignment = aff.assignPartitions(new GridAffinityFunctionContextImpl(
+                        discoCache.state().baselineTopology().createBaselineView(sorted, nodeFilter),
+                        prevAssignment, discoEvt, topVer, backups));
+
+                assignment = currentBaselineAssignment(topVer);
+            }
+            else if (hasBaseline && changedBaseline) {
+                baselineAssignment = aff.assignPartitions(new GridAffinityFunctionContextImpl(
+                    discoCache.state().baselineTopology().createBaselineView(sorted, nodeFilter),
+                    prevAssignment, discoEvt, topVer, backups));
+
+                assignment = currentBaselineAssignment(topVer);
+            }
+            else
+                assignment = aff.assignPartitions(new GridAffinityFunctionContextImpl(sorted, prevAssignment,
+                    discoEvt, topVer, backups));
+        }
+        else {
+            if (hasBaseline) {
+                baselineAssignment = aff.assignPartitions(new GridAffinityFunctionContextImpl(
+                    discoCache.state().baselineTopology().createBaselineView(sorted, nodeFilter),
+                    prevAssignment, discoEvt, topVer, backups));
+
+                assignment = currentBaselineAssignment(topVer);
+            }
             else
                 assignment = aff.assignPartitions(new GridAffinityFunctionContextImpl(sorted, prevAssignment,
                     discoEvt, topVer, backups));
         }
-        else
-            assignment = aff.assignPartitions(new GridAffinityFunctionContextImpl(sorted, prevAssignment, discoEvt,
-                topVer, backups));
 
         assert assignment != null;
 
         idealAssignment = assignment;
 
+        if (hasBaseline) {
+            baselineTopology = discoCache.state().baselineTopology();
+            assert baselineAssignment != null;
+        }
+        else {
+            baselineTopology = null;
+            baselineAssignment = null;
+        }
+
         if (locCache)
             initialize(topVer, assignment);
 
@@ -305,6 +354,41 @@ public class GridAffinityAssignmentCache {
     }
 
     /**
+     * @param topVer Topology version.
+     * @return Baseline assignment with filtered out offline nodes.
+     */
+    private List<List<ClusterNode>> currentBaselineAssignment(AffinityTopologyVersion topVer) {
+        Map<Object, ClusterNode> alives = new HashMap<>();
+
+        for (ClusterNode node : ctx.discovery().nodes(topVer)) {
+            if (!node.isClient() && !node.isDaemon())
+                alives.put(node.consistentId(), node);
+        }
+
+        List<List<ClusterNode>> result = new ArrayList<>(baselineAssignment.size());
+
+        for (int p = 0; p < baselineAssignment.size(); p++) {
+            List<ClusterNode> baselineMapping = baselineAssignment.get(p);
+            List<ClusterNode> currentMapping = null;
+
+            for (ClusterNode node : baselineMapping) {
+                ClusterNode aliveNode = alives.get(node.consistentId());
+
+                if (aliveNode != null) {
+                    if (currentMapping == null)
+                        currentMapping = new ArrayList<>();
+
+                    currentMapping.add(aliveNode);
+                }
+            }
+
+            result.add(p, currentMapping != null ? currentMapping : Collections.<ClusterNode>emptyList());
+        }
+
+        return result;
+    }
+
+    /**
      * Copies previous affinity assignment when discovery event does not cause affinity assignment changes
      * (e.g. client node joins on leaves).
      *

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 6347d03..63aba98 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -126,9 +126,6 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
     /** */
     private final ThreadLocal<ClientCacheChangeDiscoveryMessage> clientCacheChanges = new ThreadLocal<>();
 
-    /** Caches initialized flag (initialized when join activate cluster or after activation. */
-    private boolean cachesInitialized;
-
     /** Discovery listener. */
     private final GridLocalEventListener discoLsnr = new GridLocalEventListener() {
         @Override public void onEvent(Event evt) {
@@ -167,24 +164,9 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
         if (state.transition() || !state.active())
             return;
 
-        if (type == EVT_NODE_JOINED && node.isLocal()) {
-            // Clean-up in case of client reconnect.
-            caches.clear();
-
+        if (type == EVT_NODE_JOINED && node.isLocal())
             lastAffVer = null;
 
-            caches.init(cctx.cache().cacheGroupDescriptors(), cctx.cache().cacheDescriptors());
-
-            cachesInitialized = true;
-        }
-        else if (customMsg instanceof ChangeGlobalStateFinishMessage) {
-            if (!cachesInitialized && ((ChangeGlobalStateFinishMessage)customMsg).clusterActive()) {
-                caches.init(cctx.cache().cacheGroupDescriptors(), cctx.cache().cacheDescriptors());
-
-                cachesInitialized = true;
-            }
-        }
-
         if (!CU.clientNode(node) && (type == EVT_NODE_FAILED || type == EVT_NODE_JOINED || type == EVT_NODE_LEFT)) {
             synchronized (mux) {
                 assert lastAffVer == null || topVer.compareTo(lastAffVer) > 0;
@@ -195,6 +177,19 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
     }
 
     /**
+     * Must be called from exchange thread.
+     */
+    public void initCachesOnLocalJoin(
+        Map<Integer, CacheGroupDescriptor> cacheGroupDescriptors,
+        Map<String, DynamicCacheDescriptor> cacheDescriptors
+    ) {
+        // Clean-up in case of client reconnect.
+        caches.clear();
+
+        caches.init(cacheGroupDescriptors, cacheDescriptors);
+    }
+
+    /**
      * Callback invoked from discovery thread when discovery custom message is received.
      *
      * @param msg Customer message.
@@ -319,6 +314,18 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
     }
 
     /**
+     * @return Group IDs.
+     */
+    public Set<Integer> waitGroups() {
+        synchronized (mux) {
+            if (waitInfo == null || !waitInfo.topVer.equals(lastAffVer) )
+                return Collections.emptySet();
+
+            return new HashSet<>(waitInfo.waitGrps.keySet());
+        }
+    }
+
+    /**
      * @param waitInfo Cache rebalance information.
      * @return Message.
      */
@@ -706,6 +713,20 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
     }
 
     /**
+     * @param cctx Stopped cache context.
+     */
+    public void stopCacheOnReconnect(GridCacheContext cctx) {
+        caches.registeredCaches.remove(cctx.cacheId());
+    }
+
+    /**
+     * @param grpCtx Stopped cache group context.
+     */
+    public void stopCacheGroupOnReconnect(CacheGroupContext grpCtx) {
+        caches.registeredGrps.remove(grpCtx.groupId());
+    }
+
+    /**
      * Called on exchange initiated for cache start/stop request.
      *
      * @param fut Exchange future.
@@ -1412,7 +1433,8 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
      * @param crd Coordinator flag.
      * @throws IgniteCheckedException If failed.
      */
-    public void onServerJoin(final GridDhtPartitionsExchangeFuture fut, boolean crd) throws IgniteCheckedException {
+    public void onServerJoin(final GridDhtPartitionsExchangeFuture fut, boolean crd)
+        throws IgniteCheckedException {
         assert !fut.firstEvent().eventNode().isClient();
 
         boolean locJoin = fut.firstEvent().eventNode().isLocal();
@@ -1450,6 +1472,28 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
     }
 
     /**
+     * @param fut Exchange future
+     * @param crd Coordinator flag.
+     * @throws IgniteCheckedException If failed.
+     */
+    public void onBaselineTopologyChanged(final GridDhtPartitionsExchangeFuture fut, boolean crd) throws IgniteCheckedException {
+        assert !fut.firstEvent().eventNode().isClient();
+
+        WaitRebalanceInfo waitRebalanceInfo = initAffinityOnNodeJoin(fut, crd);
+
+        this.waitInfo = waitRebalanceInfo != null && !waitRebalanceInfo.empty() ? waitRebalanceInfo : null;
+
+        WaitRebalanceInfo info = this.waitInfo;
+
+        if (crd) {
+            if (log.isDebugEnabled()) {
+                log.debug("Computed new affinity after node join [topVer=" + fut.initialVersion() +
+                    ", waitGrps=" + (info != null ? groupNames(info.waitGrps.keySet()) : null) + ']');
+            }
+        }
+    }
+
+    /**
      * @param grpIds Cache group IDs.
      * @return Cache names.
      */
@@ -1892,7 +1936,6 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
      * @param rebalanceInfo Rebalance information.
      * @param latePrimary If {@code true} delays primary assignment if it is not owner.
      * @param affCache Already calculated assignments (to reduce data stored in history).
-     * @throws IgniteCheckedException If failed.
      */
     private void initAffinityOnNodeJoin(
         ExchangeDiscoveryEvents evts,
@@ -1900,9 +1943,8 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
         GridAffinityAssignmentCache aff,
         WaitRebalanceInfo rebalanceInfo,
         boolean latePrimary,
-        Map<Object, List<List<ClusterNode>>> affCache)
-        throws IgniteCheckedException
-    {
+        Map<Object, List<List<ClusterNode>>> affCache
+    ) {
         if (addedOnExchnage) {
             if (!aff.lastVersion().equals(evts.topologyVersion()))
                 calculateAndInit(evts, aff, evts.topologyVersion());

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheJoinNodeDiscoveryData.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheJoinNodeDiscoveryData.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheJoinNodeDiscoveryData.java
index ecfa8bc..6d2688c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheJoinNodeDiscoveryData.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheJoinNodeDiscoveryData.java
@@ -51,7 +51,7 @@ public class CacheJoinNodeDiscoveryData implements Serializable {
      * @param templates Templates.
      * @param startCaches {@code True} if required to start all caches on joining node.
      */
-   public CacheJoinNodeDiscoveryData(
+    public CacheJoinNodeDiscoveryData(
         IgniteUuid cacheDeploymentId,
         Map<String, CacheJoinNodeDiscoveryData.CacheInfo> caches,
         Map<String, CacheJoinNodeDiscoveryData.CacheInfo> templates,

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
index 0c4fc68..3b5a3a5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
@@ -99,7 +99,7 @@ class ClusterCachesInfo {
     private GridData gridData;
 
     /** */
-    private List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> locJoinStartCaches = Collections.emptyList();
+    private LocalJoinCachesContext locJoinCachesCtx;
 
     /** */
     private Map<String, T2<CacheConfiguration, NearCacheConfiguration>> locCfgsForActivation = Collections.emptyMap();
@@ -805,13 +805,13 @@ class ClusterCachesInfo {
      *
      * @return Caches to be started when this node starts.
      */
-    @NotNull public List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> cachesToStartOnLocalJoin() {
+    @Nullable public LocalJoinCachesContext localJoinCachesContext() {
         if (ctx.isDaemon())
-            return Collections.emptyList();
+            return null;
 
-        List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> result = locJoinStartCaches;
+        LocalJoinCachesContext result = locJoinCachesCtx;
 
-        locJoinStartCaches = Collections.emptyList();
+        locJoinCachesCtx = null;
 
         return result;
     }
@@ -888,7 +888,7 @@ class ClusterCachesInfo {
                 if (gridData == null) { // First node starts.
                     assert joinDiscoData != null;
 
-                    initStartCachesForLocalJoin(true);
+                    initStartCachesForLocalJoin(true, false);
                 }
             }
         }
@@ -1090,10 +1090,8 @@ class ClusterCachesInfo {
 
         gridData = new GridData(joinDiscoData, cachesData, conflictErr);
 
-        if (!disconnectedState())
-            initStartCachesForLocalJoin(false);
-        else
-            locJoinStartCaches = Collections.emptyList();
+        if (cachesOnDisconnect == null || cachesOnDisconnect.clusterActive())
+            initStartCachesForLocalJoin(false, disconnectedState());
     }
 
     /**
@@ -1102,8 +1100,8 @@ class ClusterCachesInfo {
      *
      * @param firstNode {@code True} if first node in cluster starts.
      */
-    private void initStartCachesForLocalJoin(boolean firstNode) {
-        assert F.isEmpty(locJoinStartCaches) : locJoinStartCaches;
+    private void initStartCachesForLocalJoin(boolean firstNode, boolean reconnect) {
+        assert locJoinCachesCtx == null : locJoinCachesCtx;
 
         if (ctx.state().clusterState().transition()) {
             joinOnTransition = true;
@@ -1112,7 +1110,7 @@ class ClusterCachesInfo {
         }
 
         if (joinDiscoData != null) {
-            locJoinStartCaches = new ArrayList<>();
+            List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> locJoinStartCaches = new ArrayList<>();
             locCfgsForActivation = new HashMap<>();
 
             boolean active = ctx.state().clusterState().active();
@@ -1123,6 +1121,9 @@ class ClusterCachesInfo {
 
                 CacheConfiguration<?, ?> cfg = desc.cacheConfiguration();
 
+                if (reconnect && surviveReconnect(cfg.getName()) && cachesOnDisconnect.state.active() && active)
+                    continue;
+
                 CacheJoinNodeDiscoveryData.CacheInfo locCfg = joinDiscoData.caches().get(cfg.getName());
 
                 NearCacheConfiguration nearCfg = null;
@@ -1159,6 +1160,11 @@ class ClusterCachesInfo {
                         locCfgsForActivation.put(desc.cacheName(), new T2<>(desc.cacheConfiguration(), nearCfg));
                 }
             }
+
+            locJoinCachesCtx = new LocalJoinCachesContext(
+                locJoinStartCaches,
+                new HashMap<>(registeredCacheGrps),
+                new HashMap<>(registeredCaches));
         }
     }
 
@@ -1167,7 +1173,7 @@ class ClusterCachesInfo {
      */
     public void onStateChangeFinish(ChangeGlobalStateFinishMessage msg) {
         if (joinOnTransition) {
-            initStartCachesForLocalJoin(false);
+            initStartCachesForLocalJoin(false, false);
 
             joinOnTransition = false;
         }
@@ -1179,10 +1185,13 @@ class ClusterCachesInfo {
      * @return Exchange action.
      * @throws IgniteCheckedException If configuration validation failed.
      */
-    public ExchangeActions onStateChangeRequest(ChangeGlobalStateMessage msg, AffinityTopologyVersion topVer)
+    public ExchangeActions onStateChangeRequest(ChangeGlobalStateMessage msg, AffinityTopologyVersion topVer, DiscoveryDataClusterState curState)
         throws IgniteCheckedException {
         ExchangeActions exchangeActions = new ExchangeActions();
 
+        if (msg.activate() == curState.active())
+            return exchangeActions;
+
         if (msg.activate()) {
             for (DynamicCacheDescriptor desc : orderedCaches(CacheComparators.DIRECT)) {
                 desc.startTopologyVersion(topVer);
@@ -1657,6 +1666,9 @@ class ClusterCachesInfo {
         Set<String> stoppedCaches = new HashSet<>();
         Set<Integer> stoppedCacheGrps = new HashSet<>();
 
+        Set<String> survivedCaches = new HashSet<>();
+        Set<Integer> survivedCacheGrps = new HashSet<>();
+
         if (!active) {
             joinOnTransition = transition;
 
@@ -1698,8 +1710,11 @@ class ClusterCachesInfo {
 
                 if (stopped)
                     stoppedCacheGrps.add(locDesc.groupId());
-                else
+                else {
                     assert locDesc.groupId() == desc.groupId();
+
+                    survivedCacheGrps.add(locDesc.groupId());
+                }
             }
 
             for (Map.Entry<String, DynamicCacheDescriptor> e : cachesOnDisconnect.caches.entrySet()) {
@@ -1719,10 +1734,20 @@ class ClusterCachesInfo {
 
                 if (stopped)
                     stoppedCaches.add(cacheName);
+                else
+                    survivedCaches.add(cacheName);
+            }
+
+            if (locJoinCachesCtx != null) {
+                locJoinCachesCtx.removeSurvivedCacheGroups(survivedCacheGrps);
+                locJoinCachesCtx.removeSurvivedCaches(survivedCaches);
+
+                if (locJoinCachesCtx.isEmpty())
+                    locJoinCachesCtx = null;
             }
 
             if (!cachesOnDisconnect.clusterActive())
-                initStartCachesForLocalJoin(false);
+                initStartCachesForLocalJoin(false, true);
         }
 
         if (clientReconnectReqs != null) {

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
index 91ad003..d84f8d8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
@@ -50,6 +50,9 @@ public class ExchangeActions {
     private Map<String, CacheActionData> cachesToResetLostParts;
 
     /** */
+    private LocalJoinCachesContext locJoinCtx;
+
+    /** */
     private StateChangeRequest stateChangeReq;
 
     /**
@@ -194,14 +197,21 @@ public class ExchangeActions {
      * @return {@code True} if has deactivate request.
      */
     public boolean deactivate() {
-        return stateChangeReq != null && !stateChangeReq.activate();
+        return stateChangeReq != null && stateChangeReq.activeChanged() && !stateChangeReq.activate();
     }
 
     /**
      * @return {@code True} if has activate request.
      */
     public boolean activate() {
-        return stateChangeReq != null && stateChangeReq.activate();
+        return stateChangeReq != null && stateChangeReq.activeChanged() && stateChangeReq.activate();
+    }
+
+    /**
+     * @return {@code True} if has baseline topology change request.
+     */
+    public boolean changedBaseline() {
+        return stateChangeReq != null && !stateChangeReq.activeChanged();
     }
 
     /**
@@ -341,7 +351,22 @@ public class ExchangeActions {
             F.isEmpty(cacheGrpsToStart) &&
             F.isEmpty(cacheGrpsToStop) &&
             F.isEmpty(cachesToResetLostParts) &&
-            stateChangeReq == null;
+            stateChangeReq == null &&
+            locJoinCtx == null;
+    }
+
+    /**
+     * @param locJoinCtx Caches local join context.
+     */
+    public void localJoinContext(LocalJoinCachesContext locJoinCtx) {
+        this.locJoinCtx = locJoinCtx;
+    }
+
+    /**
+     * @return Caches local join context.
+     */
+    public LocalJoinCachesContext localJoinContext() {
+        return locJoinCtx;
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeDiscoveryEvents.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeDiscoveryEvents.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeDiscoveryEvents.java
index d4fbe60..0e7e01c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeDiscoveryEvents.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeDiscoveryEvents.java
@@ -125,22 +125,17 @@ public class ExchangeDiscoveryEvents {
         this.lastEvt = evt;
         this.discoCache = cache;
 
-        if (evt.type() != EVT_DISCOVERY_CUSTOM_EVT) {
-            ClusterNode node = evt.eventNode();
+        ClusterNode node = evt.eventNode();
 
-            if (!CU.clientNode(node)) {
-                lastSrvEvt = evt;
+        if (!CU.clientNode(node)) {
+            lastSrvEvt = evt;
 
-                srvEvtTopVer = new AffinityTopologyVersion(evt.topologyVersion(), 0);
+            srvEvtTopVer = new AffinityTopologyVersion(evt.topologyVersion(), 0);
 
-                if (evt.type()== EVT_NODE_JOINED)
-                    srvJoin = true;
-                else {
-                    assert evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED : evt;
-
-                    srvLeft = !CU.clientNode(node);
-                }
-            }
+            if (evt.type()== EVT_NODE_JOINED)
+                srvJoin = true;
+            else if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED)
+                srvLeft = !CU.clientNode(node);
         }
     }
 
@@ -160,6 +155,14 @@ public class ExchangeDiscoveryEvents {
     }
 
     /**
+     * @param evt Event.
+     * @return {@code True} if given event is {@link EventType#EVT_NODE_JOINED}.
+     */
+    public static boolean serverJoinEvent(DiscoveryEvent evt) {
+        return  (evt.type() == EVT_NODE_JOINED && !CU.clientNode(evt.eventNode()));
+    }
+
+    /**
      * @return Discovery data cache for last event.
      */
     public DiscoCache discoveryCache() {

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 4370e75..50b3909 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -826,6 +826,8 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
                 }
 
                 if (e != null) {
+                    ctx.shared().database().checkpointReadLock();
+
                     try {
                         cacheVal = e.peek(modes.heap, modes.offheap, topVer, plc);
                     }
@@ -837,6 +839,8 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
                     }
                     finally {
                         ctx0.evicts().touch(e, null);
+
+                        ctx.shared().database().checkpointReadUnlock();
                     }
                 }
 
@@ -2052,7 +2056,15 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
 
                                             try {
                                                 ctx.shared().database().ensureFreeSpace(ctx.dataRegion());
+                                            }
+                                            catch (IgniteCheckedException e) {
+                                                // Wrap errors (will be unwrapped).
+                                                throw new GridClosureException(e);
+                                            }
 
+                                            ctx.shared().database().checkpointReadLock();
+
+                                            try {
                                                 entry = entryEx(key);
 
                                                 entry.unswap();
@@ -2097,6 +2109,9 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
                                                 // Wrap errors (will be unwrapped).
                                                 throw new GridClosureException(e);
                                             }
+                                            finally {
+                                                ctx.shared().database().checkpointReadUnlock();
+                                            }
                                         }
                                     }
                                 });
@@ -4454,6 +4469,8 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
 
         GridCacheVersion obsoleteVer = ctx.versions().next();
 
+        ctx.shared().database().checkpointReadLock();
+
         try {
             KeyCacheObject cacheKey = ctx.toCacheKeyObject(key);
 
@@ -4468,6 +4485,9 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
         catch (IgniteCheckedException ex) {
             U.error(log, "Failed to clearLocally entry for key: " + key, ex);
         }
+        finally {
+            ctx.shared().database().checkpointReadUnlock();
+        }
 
         return false;
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
index 0cdf669..103a976 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
@@ -62,6 +62,7 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.datastructures.CacheDataStructuresManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTransactionalCacheAdapter;
@@ -2032,24 +2033,48 @@ public class GridCacheContext<K, V> implements Externalizable {
     }
 
     /**
+     * Checks if local reads are allowed for the given partition and reserves the partition when needed. If this
+     * method returns {@code true}, then {@link #releaseForFastLocalGet(int, AffinityTopologyVersion)} method
+     * must be called after the read is completed.
+     *
      * @param part Partition.
-     * @param affNodes Affinity nodes.
      * @param topVer Topology version.
      * @return {@code True} if cache 'get' operation is allowed to get entry locally.
      */
-    public boolean allowFastLocalRead(int part, List<ClusterNode> affNodes, AffinityTopologyVersion topVer) {
-        boolean result = affinityNode() && rebalanceEnabled() && hasPartition(part, affNodes, topVer);
+    public boolean reserveForFastLocalGet(int part, AffinityTopologyVersion topVer) {
+        boolean result = affinityNode() && rebalanceEnabled() && checkAndReservePartition(part, topVer);
 
         // When persistence is enabled, only reading from partitions with OWNING state is allowed.
         assert !result || !group().persistenceEnabled() ||
             topology().partitionState(localNodeId(), part) == OWNING :
-            "result = " + result + ", persistenceEnabled = " + group().persistenceEnabled() +
-                ", partitionState = " + topology().partitionState(localNodeId(), part);
+            "result=" + result + ", persistenceEnabled=" + group().persistenceEnabled() +
+                ", partitionState=" + topology().partitionState(localNodeId(), part) +
+                ", replicated=" + isReplicated();
 
         return result;
     }
 
     /**
+     * Releases the partition that was reserved by a call to
+     * {@link #reserveForFastLocalGet(int, AffinityTopologyVersion)}.
+     *
+     * @param part Partition to release.
+     * @param topVer Topology version.
+     */
+    public void releaseForFastLocalGet(int part, AffinityTopologyVersion topVer) {
+        assert affinityNode();
+
+        if (!isReplicated() || group().persistenceEnabled()) {
+            GridDhtLocalPartition locPart = topology().localPartition(part, topVer, false);
+
+            assert locPart != null && locPart.state() == OWNING : "partition evicted after reserveForFastLocalGet " +
+                "[part=" + part + ", locPart=" + locPart + ", topVer=" + topVer + ']';
+
+            locPart.release();
+        }
+    }
+
+    /**
      * Checks if it is possible to directly read data memory without entry creation (this
      * is optimization to avoid unnecessary blocking synchronization on cache entry).
      *
@@ -2063,17 +2088,44 @@ public class GridCacheContext<K, V> implements Externalizable {
 
     /**
      * @param part Partition.
-     * @param affNodes Affinity nodes.
      * @param topVer Topology version.
      * @return {@code True} if partition is available locally.
      */
-    private boolean hasPartition(int part, List<ClusterNode> affNodes, AffinityTopologyVersion topVer) {
+    private boolean checkAndReservePartition(int part, AffinityTopologyVersion topVer) {
         assert affinityNode();
 
         GridDhtPartitionTopology top = topology();
 
-        return (top.rebalanceFinished(topVer) && (isReplicated() || affNodes.contains(locNode)))
-            || (top.partitionState(localNodeId(), part) == OWNING);
+        if (isReplicated() && !group().persistenceEnabled()) {
+            boolean rebFinished = top.rebalanceFinished(topVer);
+
+            if (rebFinished)
+                return true;
+
+            GridDhtLocalPartition locPart = top.localPartition(part, topVer, false, false);
+
+            // No need to reserve a partition for REPLICATED cache because this partition cannot be evicted.
+            return locPart != null && locPart.state() == OWNING;
+        }
+        else {
+            GridDhtLocalPartition locPart = top.localPartition(part, topVer, false, false);
+
+            if (locPart != null && locPart.reserve()) {
+                boolean canRead = true;
+
+                try {
+                    canRead = locPart.state() == OWNING;
+
+                    return canRead;
+                }
+                finally {
+                    if (!canRead)
+                        locPart.release();
+                }
+            }
+            else
+                return false;
+        }
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 345c2e4..cb36acd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -35,6 +35,7 @@ import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.eviction.EvictableEntry;
 import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -943,7 +944,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
 
         // Lock should be held by now.
         if (!cctx.isAll(this, filter))
-            return new GridCacheUpdateTxResult(false, null);
+            return new GridCacheUpdateTxResult(false, null, null);
 
         final GridCacheVersion newVer;
 
@@ -951,6 +952,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
 
         Object key0 = null;
         Object val0 = null;
+        WALPointer logPtr = null;
 
         long updateCntr0;
 
@@ -966,7 +968,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
 
                 // It is possible that 'get' could load more recent value.
                 if (!((GridNearCacheEntry)this).recordDhtVersion(dhtVer))
-                    return new GridCacheUpdateTxResult(false, null);
+                    return new GridCacheUpdateTxResult(false, null, logPtr);
             }
 
             assert tx == null || (!tx.local() && tx.onePhaseCommit()) || tx.ownsLock(this) :
@@ -1002,7 +1004,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
                 key0 = e.key();
 
                 if (interceptorVal == null)
-                    return new GridCacheUpdateTxResult(false, (CacheObject)cctx.unwrapTemporary(old));
+                    return new GridCacheUpdateTxResult(false, (CacheObject)cctx.unwrapTemporary(old), logPtr);
                 else if (interceptorVal != val0)
                     val0 = cctx.unwrapTemporary(interceptorVal);
 
@@ -1044,6 +1046,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
             if (updateCntr != null && updateCntr != 0)
                 updateCntr0 = updateCntr;
 
+            if (tx != null && cctx.group().persistenceEnabled())
+                logPtr = logTxUpdate(tx, val, expireTime, updateCntr0);
+
             update(val, expireTime, ttl, newVer, true);
 
             drReplicate(drType, val, newVer, topVer);
@@ -1104,8 +1109,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
         if (intercept)
             cctx.config().getInterceptor().onAfterPut(new CacheLazyEntry(cctx, key, key0, val, val0, keepBinary, updateCntr0));
 
-        return valid ? new GridCacheUpdateTxResult(true, retval ? old : null, updateCntr0) :
-            new GridCacheUpdateTxResult(false, null);
+        return valid ? new GridCacheUpdateTxResult(true, retval ? old : null, updateCntr0, logPtr) :
+            new GridCacheUpdateTxResult(false, null, logPtr);
     }
 
     /**
@@ -1146,7 +1151,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
 
         // Lock should be held by now.
         if (!cctx.isAll(this, filter))
-            return new GridCacheUpdateTxResult(false, null);
+            return new GridCacheUpdateTxResult(false, null, null);
 
         GridCacheVersion obsoleteVer = null;
 
@@ -1158,6 +1163,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
 
         long updateCntr0;
 
+        WALPointer logPtr = null;
+
         boolean deferred;
 
         boolean marked = false;
@@ -1172,7 +1179,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
 
                 // It is possible that 'get' could load more recent value.
                 if (!((GridNearCacheEntry)this).recordDhtVersion(dhtVer))
-                    return new GridCacheUpdateTxResult(false, null);
+                    return new GridCacheUpdateTxResult(false, null, logPtr);
             }
 
             assert tx == null || (!tx.local() && tx.onePhaseCommit()) || tx.ownsLock(this) :
@@ -1200,7 +1207,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
                 if (cctx.cancelRemove(interceptRes)) {
                     CacheObject ret = cctx.toCacheObject(cctx.unwrapTemporary(interceptRes.get2()));
 
-                    return new GridCacheUpdateTxResult(false, ret);
+                    return new GridCacheUpdateTxResult(false, ret, logPtr);
                 }
             }
 
@@ -1228,6 +1235,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
             if (updateCntr != null && updateCntr != 0)
                 updateCntr0 = updateCntr;
 
+            if (tx != null && cctx.group().persistenceEnabled())
+                logPtr = logTxUpdate(tx, null, 0, updateCntr0);
+
             drReplicate(drType, null, newVer, topVer);
 
             if (metrics && cctx.statisticsEnabled())
@@ -1325,10 +1335,10 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
             else
                 ret = old;
 
-            return new GridCacheUpdateTxResult(true, ret, updateCntr0);
+            return new GridCacheUpdateTxResult(true, ret, updateCntr0, logPtr);
         }
         else
-            return new GridCacheUpdateTxResult(false, null);
+            return new GridCacheUpdateTxResult(false, null, logPtr);
     }
 
     /**
@@ -3549,6 +3559,39 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
     }
 
     /**
+     * @param tx Transaction.
+     * @param val Value.
+     * @param expireTime Expire time (or 0 if not applicable).
+     * @param updCntr Update counter.
+     * @throws IgniteCheckedException In case of log failure.
+     */
+    protected WALPointer logTxUpdate(IgniteInternalTx tx, CacheObject val, long expireTime, long updCntr)
+        throws IgniteCheckedException {
+        assert cctx.transactional();
+
+        if (tx.local()) { // For remote tx we log all updates in batch: GridDistributedTxRemoteAdapter.commitIfLocked()
+            GridCacheOperation op;
+            if (val == null)
+                op = GridCacheOperation.DELETE;
+            else
+                op = this.val == null ? GridCacheOperation.CREATE : GridCacheOperation.UPDATE;
+
+            return cctx.shared().wal().log(new DataRecord(new DataEntry(
+                cctx.cacheId(),
+                key,
+                val,
+                op,
+                tx.nearXidVersion(),
+                tx.writeVersion(),
+                expireTime,
+                key.partition(),
+                updCntr)));
+        }
+        else
+            return null;
+    }
+
+    /**
      * Removes value from offheap.
      *
      * @throws IgniteCheckedException If failed.

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index fe9ed29..7107125 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -81,6 +81,8 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.Gri
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionHistorySuppliersMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionsToReloadMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.RebalanceReassignExchangeTask;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.StopCachesOnClientReconnectExchangeTask;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager;
@@ -254,7 +256,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
 
                 if (cache.state().transition()) {
                     if (log.isDebugEnabled())
-                        log.debug("Add pending event: " + evt);
+                        log.debug("Adding pending event: " + evt);
 
                     pendingEvts.add(new PendingDiscoveryEvent(evt, cache));
                 }
@@ -290,6 +292,9 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
      * @param cache Discovery data cache.
      */
     private void processEventInactive(DiscoveryEvent evt, DiscoCache cache) {
+        // Clean local join caches context.
+        cctx.cache().localJoinCachesContext();
+
         if (log.isDebugEnabled())
             log.debug("Ignore event, cluster is inactive: " + evt);
    }
@@ -417,7 +422,19 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
 
             exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt);
 
-            exchFut = exchangeFuture(exchId, evt, cache, null, null);
+            ExchangeActions exchActs = null;
+
+            if (evt.type() == EVT_NODE_JOINED && evt.eventNode().isLocal()) {
+                LocalJoinCachesContext locJoinCtx = cctx.cache().localJoinCachesContext();
+
+                if (locJoinCtx != null) {
+                    exchActs = new ExchangeActions();
+
+                    exchActs.localJoinContext(locJoinCtx);
+                }
+            }
+
+            exchFut = exchangeFuture(exchId, evt, cache, exchActs, null);
         }
         else {
             DiscoveryCustomMessage customMsg = ((DiscoveryCustomEvent)evt).customMessage();
@@ -555,7 +572,12 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
 
             GridDhtPartitionExchangeId exchId = initialExchangeId();
 
-            fut = exchangeFuture(exchId, discoEvt, discoCache, null, null);
+            fut = exchangeFuture(
+                exchId,
+                reconnect ? null : discoEvt,
+                reconnect ? null : discoCache,
+                null,
+                null);
         }
         else if (reconnect)
             reconnectExchangeFut.onDone();
@@ -907,6 +929,16 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
     }
 
     /**
+     * @param caches Caches to stop.
+     * @return Future that will be completed when caches are stopped from the exchange thread.
+     */
+    public IgniteInternalFuture<Void> deferStopCachesOnClientReconnect(Collection<GridCacheAdapter> caches) {
+        assert cctx.discovery().localNode().isClient();
+
+        return exchWorker.deferStopCachesOnClientReconnect(caches);
+    }
+
+    /**
      * Schedules next full partitions update.
      */
     public void scheduleResendPartitions() {
@@ -926,6 +958,10 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
      * for non coordinator -  {@link GridDhtPartitionsSingleMessage SingleMessages} send
      */
     private void refreshPartitions() {
+        // TODO https://issues.apache.org/jira/browse/IGNITE-6857
+        if (cctx.snapshot().snapshotOperationInProgress())
+            return;
+
         ClusterNode oldest = cctx.discovery().oldestAliveServerNode(AffinityTopologyVersion.NONE);
 
         if (oldest == null) {
@@ -2042,6 +2078,17 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
         }
 
         /**
+         * @param caches Caches to stop.
+         */
+        IgniteInternalFuture<Void> deferStopCachesOnClientReconnect(Collection<GridCacheAdapter> caches) {
+            StopCachesOnClientReconnectExchangeTask task = new StopCachesOnClientReconnectExchangeTask(caches);
+
+            futQ.add(task);
+
+            return task;
+        }
+
+        /**
          * @param exchFut Exchange future.
          */
         void addExchangeFuture(GridDhtPartitionsExchangeFuture exchFut) {
@@ -2169,7 +2216,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
             int cnt = 0;
 
             for (CachePartitionExchangeWorkerTask task : futQ) {
-                if (isExchangeTask(task)) {
+                if (task instanceof GridDhtPartitionsExchangeFuture) {
                     U.warn(log, ">>> " + ((GridDhtPartitionsExchangeFuture)task).shortInfo());
 
                     if (++cnt == 10)
@@ -2344,13 +2391,17 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
                         if (!cctx.kernalContext().clientNode()) {
                             assignsMap = new HashMap<>();
 
-                            for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
+                            IgniteCacheSnapshotManager snp = cctx.snapshot();
+
+                            for (final CacheGroupContext grp : cctx.cache().cacheGroups()) {
                                 long delay = grp.config().getRebalanceDelay();
 
+                                boolean disableRebalance = snp.partitionsAreFrozen(grp);
+
                                 GridDhtPreloaderAssignments assigns = null;
 
                                 // Don't delay for dummy reassigns to avoid infinite recursion.
-                                if (delay == 0 || forcePreload)
+                                if ((delay == 0 || forcePreload) && !disableRebalance)
                                     assigns = grp.preloader().assign(exchId, exchFut);
 
                                 assignsMap.put(grp.groupId(), assigns);

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index e635e62..f234ecc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -81,6 +81,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalP
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedCache;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.StopCachesOnClientReconnectExchangeTask;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearAtomicCache;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTransactionalCache;
 import org.apache.ignite.internal.processors.cache.dr.GridCacheDrManager;
@@ -92,6 +93,7 @@ import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabase
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
@@ -360,6 +362,13 @@ public class GridCacheProcessor extends GridProcessorAdapter {
 
             processStatisticsModeChange(task0.message());
         }
+        else if (task instanceof StopCachesOnClientReconnectExchangeTask) {
+            StopCachesOnClientReconnectExchangeTask task0 = (StopCachesOnClientReconnectExchangeTask)task;
+
+            stopCachesOnClientReconnect(task0.stoppedCaches());
+
+            task0.onDone();
+        }
         else
             U.warn(log, "Unsupported custom exchange task: " + task);
     }
@@ -1085,25 +1094,10 @@ public class GridCacheProcessor extends GridProcessorAdapter {
         for (GridCacheAdapter cache : reconnected)
             cache.context().gate().reconnected(false);
 
-        IgniteInternalFuture<?> stopFut = null;
-
-        if (!stoppedCaches.isEmpty()) {
-            stopFut = ctx.closure().runLocalSafe(new Runnable() {
-                @Override public void run() {
-                    for (GridCacheAdapter cache : stoppedCaches) {
-                        CacheGroupContext grp = cache.context().group();
-
-                        onKernalStop(cache, true);
-                        stopCache(cache, true, false);
-
-                        if (!grp.hasCaches())
-                            stopCacheGroup(grp);
-                    }
-                }
-            });
-        }
+        if (!stoppedCaches.isEmpty())
+            return sharedCtx.exchange().deferStopCachesOnClientReconnect(stoppedCaches);
 
-        return stopFut;
+        return null;
     }
 
     /**
@@ -1160,12 +1154,12 @@ public class GridCacheProcessor extends GridProcessorAdapter {
 
         if (log.isInfoEnabled()) {
             log.info("Started cache [name=" + cfg.getName() +
-                ", id=" + cacheCtx.cacheId() +
+                ", id="+cacheCtx.cacheId() +
                 (cfg.getGroupName() != null ? ", group=" + cfg.getGroupName() : "") +
                 ", memoryPolicyName=" + memPlcName +
                 ", mode=" + cfg.getCacheMode() +
                 ", atomicity=" + cfg.getAtomicityMode() +
-                ", backups=" + cfg.getBackups() + ']');
+                ", backups=" + cfg.getBackups() +']');
         }
     }
 
@@ -1178,69 +1172,71 @@ public class GridCacheProcessor extends GridProcessorAdapter {
     private void stopCache(GridCacheAdapter<?, ?> cache, boolean cancel, boolean destroy) {
         GridCacheContext ctx = cache.context();
 
-        if (!cache.isNear() && ctx.shared().wal() != null) {
-            try {
-                ctx.shared().wal().fsync(null);
-            }
-            catch (IgniteCheckedException e) {
-                U.error(log, "Failed to flush write-ahead log on cache stop " +
-                    "[cache=" + ctx.name() + "]", e);
+        try {
+            if (!cache.isNear() && ctx.shared().wal() != null) {
+                try {
+                    ctx.shared().wal().fsync(null);
+                } catch (IgniteCheckedException e) {
+                    U.error(log, "Failed to flush write-ahead log on cache stop " +
+                            "[cache=" + ctx.name() + "]", e);
+                }
             }
-        }
 
-        sharedCtx.removeCacheContext(ctx);
+            sharedCtx.removeCacheContext(ctx);
 
-        cache.stop();
+            cache.stop();
 
-        ctx.kernalContext().query().onCacheStop(ctx, !cache.context().group().persistenceEnabled() || destroy);
+            ctx.kernalContext().query().onCacheStop(ctx, !cache.context().group().persistenceEnabled() || destroy);
 
-        if (isNearEnabled(ctx)) {
-            GridDhtCacheAdapter dht = ctx.near().dht();
+            if (isNearEnabled(ctx)) {
+                GridDhtCacheAdapter dht = ctx.near().dht();
 
-            // Check whether dht cache has been started.
-            if (dht != null) {
-                dht.stop();
+                // Check whether dht cache has been started.
+                if (dht != null) {
+                    dht.stop();
 
-                GridCacheContext<?, ?> dhtCtx = dht.context();
+                    GridCacheContext<?, ?> dhtCtx = dht.context();
 
-                List<GridCacheManager> dhtMgrs = dhtManagers(dhtCtx);
+                    List<GridCacheManager> dhtMgrs = dhtManagers(dhtCtx);
 
-                for (ListIterator<GridCacheManager> it = dhtMgrs.listIterator(dhtMgrs.size()); it.hasPrevious(); ) {
-                    GridCacheManager mgr = it.previous();
+                    for (ListIterator<GridCacheManager> it = dhtMgrs.listIterator(dhtMgrs.size()); it.hasPrevious(); ) {
+                        GridCacheManager mgr = it.previous();
 
-                    mgr.stop(cancel, destroy);
+                        mgr.stop(cancel, destroy);
+                    }
                 }
             }
-        }
 
-        List<GridCacheManager> mgrs = ctx.managers();
+            List<GridCacheManager> mgrs = ctx.managers();
 
-        Collection<GridCacheManager> excludes = dhtExcludes(ctx);
+            Collection<GridCacheManager> excludes = dhtExcludes(ctx);
 
-        // Reverse order.
-        for (ListIterator<GridCacheManager> it = mgrs.listIterator(mgrs.size()); it.hasPrevious(); ) {
-            GridCacheManager mgr = it.previous();
+            // Reverse order.
+            for (ListIterator<GridCacheManager> it = mgrs.listIterator(mgrs.size()); it.hasPrevious(); ) {
+                GridCacheManager mgr = it.previous();
 
-            if (!excludes.contains(mgr))
-                mgr.stop(cancel, destroy);
-        }
+                if (!excludes.contains(mgr))
+                    mgr.stop(cancel, destroy);
+            }
 
-        ctx.kernalContext().continuous().onCacheStop(ctx);
+            ctx.kernalContext().continuous().onCacheStop(ctx);
 
-        ctx.kernalContext().cache().context().snapshot().onCacheStop(ctx);
+            ctx.kernalContext().cache().context().snapshot().onCacheStop(ctx);
 
-        ctx.group().stopCache(ctx, destroy);
+            ctx.group().stopCache(ctx, destroy);
 
-        U.stopLifecycleAware(log, lifecycleAwares(ctx.group(), cache.configuration(), ctx.store().configuredStore()));
+            U.stopLifecycleAware(log, lifecycleAwares(ctx.group(), cache.configuration(), ctx.store().configuredStore()));
 
-        if (log.isInfoEnabled()) {
-            if (ctx.group().sharedGroup())
-                log.info("Stopped cache [cacheName=" + cache.name() + ", group=" + ctx.group().name() + ']');
-            else
-                log.info("Stopped cache [cacheName=" + cache.name() + ']');
+            if (log.isInfoEnabled()) {
+                if (ctx.group().sharedGroup())
+                    log.info("Stopped cache [cacheName=" + cache.name() + ", group=" + ctx.group().name() + ']');
+                else
+                    log.info("Stopped cache [cacheName=" + cache.name() + ']');
+            }
+        }
+        finally {
+            cleanup(ctx);
         }
-
-        cleanup(ctx);
     }
 
     /**
@@ -1733,20 +1729,24 @@ public class GridCacheProcessor extends GridProcessorAdapter {
     /**
      * @return Caches to be started when this node starts.
      */
-    @NotNull public List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> cachesToStartOnLocalJoin() {
-        return cachesInfo.cachesToStartOnLocalJoin();
+    @Nullable public LocalJoinCachesContext localJoinCachesContext() {
+        return cachesInfo.localJoinCachesContext();
     }
 
     /**
-     * @param caches Caches to start.
+     * @param locJoinCtx Local join cache context.
      * @param exchTopVer Current exchange version.
      * @throws IgniteCheckedException If failed.
      */
-    public void startCachesOnLocalJoin(List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> caches,
-        AffinityTopologyVersion exchTopVer)
-        throws IgniteCheckedException {
-        if (!F.isEmpty(caches)) {
-            for (T2<DynamicCacheDescriptor, NearCacheConfiguration> t : caches) {
+    public void startCachesOnLocalJoin(
+        LocalJoinCachesContext locJoinCtx,
+        AffinityTopologyVersion exchTopVer
+    ) throws IgniteCheckedException {
+        if (locJoinCtx != null) {
+            sharedCtx.affinity().initCachesOnLocalJoin(
+                locJoinCtx.cacheGroupDescriptors(), locJoinCtx.cacheDescriptors());
+
+            for (T2<DynamicCacheDescriptor, NearCacheConfiguration> t : locJoinCtx.caches()) {
                 DynamicCacheDescriptor desc = t.get1();
 
                 prepareCacheStart(
@@ -2402,12 +2402,16 @@ public class GridCacheProcessor extends GridProcessorAdapter {
     /**
      * @param msg Message.
      * @param topVer Current topology version.
+     * @param curState Current cluster state.
      * @throws IgniteCheckedException If configuration validation failed.
      * @return Exchange actions.
      */
-    public ExchangeActions onStateChangeRequest(ChangeGlobalStateMessage msg, AffinityTopologyVersion topVer)
-        throws IgniteCheckedException {
-        return cachesInfo.onStateChangeRequest(msg, topVer);
+    public ExchangeActions onStateChangeRequest(
+        ChangeGlobalStateMessage msg,
+        AffinityTopologyVersion topVer,
+        DiscoveryDataClusterState curState
+    ) throws IgniteCheckedException {
+        return cachesInfo.onStateChangeRequest(msg, topVer, curState);
     }
 
     /**
@@ -2476,6 +2480,28 @@ public class GridCacheProcessor extends GridProcessorAdapter {
     }
 
     /**
+     * @param stoppedCaches Stopped caches.
+     */
+    private void stopCachesOnClientReconnect(Collection<GridCacheAdapter> stoppedCaches) {
+        assert ctx.discovery().localNode().isClient();
+
+        for (GridCacheAdapter cache : stoppedCaches) {
+            CacheGroupContext grp = cache.context().group();
+
+            onKernalStop(cache, true);
+            stopCache(cache, true, false);
+
+            sharedCtx.affinity().stopCacheOnReconnect(cache.context());
+
+            if (!grp.hasCaches()) {
+                stopCacheGroup(grp);
+
+                sharedCtx.affinity().stopCacheGroupOnReconnect(grp);
+            }
+        }
+    }
+
+    /**
      * @return {@code True} if need locally start all existing caches on client node start.
      */
     private boolean startAllCachesOnClientStart() {
@@ -4039,7 +4065,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
         if (ccfg != null) {
             cloneCheckSerializable(ccfg);
 
-            if (desc != null) {
+            if (desc != null || MetaStorage.METASTORAGE_CACHE_NAME.equals(cacheName)) {
                 if (failIfExists) {
                     throw new CacheExistsException("Failed to start cache " +
                         "(a cache with the same name is already started): " + cacheName);

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
index 461baa7..92af83b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.cache;
 
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.Nullable;
@@ -35,27 +36,34 @@ public class GridCacheUpdateTxResult {
     /** Partition idx. */
     private long updateCntr;
 
+    /** */
+    private WALPointer logPtr;
+
     /**
      * Constructor.
      *
      * @param success Success flag.
      * @param oldVal Old value (if any),
+     * @param logPtr Logger WAL pointer for the update.
      */
-    GridCacheUpdateTxResult(boolean success, @Nullable CacheObject oldVal) {
+    GridCacheUpdateTxResult(boolean success, @Nullable CacheObject oldVal, WALPointer logPtr) {
         this.success = success;
         this.oldVal = oldVal;
+        this.logPtr = logPtr;
     }
 
     /**
      * Constructor.
      *
      * @param success Success flag.
-     * @param oldVal Old value (if any),
+     * @param oldVal Old value (if any).
+     * @param logPtr Logger WAL pointer for the update.
      */
-    GridCacheUpdateTxResult(boolean success, @Nullable CacheObject oldVal, long updateCntr) {
+    GridCacheUpdateTxResult(boolean success, @Nullable CacheObject oldVal, long updateCntr, WALPointer logPtr) {
         this.success = success;
         this.oldVal = oldVal;
         this.updateCntr = updateCntr;
+        this.logPtr = logPtr;
     }
 
     /**
@@ -73,6 +81,13 @@ public class GridCacheUpdateTxResult {
     }
 
     /**
+     * @return Logged WAL pointer for the update if persistence is enabled.
+     */
+    public WALPointer loggedPointer() {
+        return logPtr;
+    }
+
+    /**
      * @return Old value.
      */
     @Nullable public CacheObject oldValue() {

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 17d599b..0d63fc0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -73,6 +73,7 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLo
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
 import org.apache.ignite.internal.processors.dr.GridDrType;
 import org.apache.ignite.internal.processors.igfs.IgfsUtils;
 import org.apache.ignite.internal.processors.query.QueryUtils;
@@ -1375,6 +1376,15 @@ public class GridCacheUtils {
     }
 
     /**
+     * @param node Node.
+     * @param discoveryDataClusterState Discovery data cluster state.
+     * @return {@code True} if node is included in BaselineTopology.
+     */
+    public static boolean baselineNode(ClusterNode node, DiscoveryDataClusterState discoveryDataClusterState) {
+        return discoveryDataClusterState.baselineTopology().consistentIds().contains(node.consistentId());
+    }
+
+    /**
      * Creates and starts store session listeners.
      *
      * @param ctx Kernal context.
@@ -1707,8 +1717,8 @@ public class GridCacheUtils {
         final AffinityTopologyVersion topVer,
         final IgniteLogger log,
         final GridCacheContext cctx,
-        final @Nullable KeyCacheObject key,
-        final @Nullable IgniteCacheExpiryPolicy expiryPlc,
+        @Nullable final KeyCacheObject key,
+        @Nullable final IgniteCacheExpiryPolicy expiryPlc,
         boolean readThrough,
         boolean skipVals
     ) {
@@ -1721,6 +1731,8 @@ public class GridCacheUtils {
                 while (true) {
                     GridCacheEntryEx entry = null;
 
+                    cctx.shared().database().checkpointReadLock();
+
                     try {
                         entry = colocated.entryEx(key, topVer);
 
@@ -1750,6 +1762,8 @@ public class GridCacheUtils {
                     finally {
                         if (entry != null)
                             cctx.evicts().touch(entry, topVer);
+
+                        cctx.shared().database().checkpointReadUnlock();
                     }
                 }
             }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index 8793729..d6f178d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -41,8 +41,8 @@ import org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapt
 import org.apache.ignite.internal.processors.cache.persistence.CacheSearchRow;
 import org.apache.ignite.internal.processors.cache.persistence.RootPage;
 import org.apache.ignite.internal.processors.cache.persistence.RowStore;
-import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl;
 import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager;
 import org.apache.ignite.internal.processors.cache.tree.CacheDataRowStore;
@@ -1175,12 +1175,12 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
             // Use grp.sharedGroup() flag since it is possible cacheId is not yet set here.
             boolean sizeWithCacheId = grp.sharedGroup();
 
-            int oldLen = FreeListImpl.getRowSize(oldRow, sizeWithCacheId);
+            int oldLen = DataPageIO.getRowSize(oldRow, sizeWithCacheId);
 
             if (oldLen > updateValSizeThreshold)
                 return false;
 
-            int newLen = FreeListImpl.getRowSize(dataRow, sizeWithCacheId);
+            int newLen = DataPageIO.getRowSize(dataRow, sizeWithCacheId);
 
             return oldLen == newLen;
         }
@@ -1565,6 +1565,7 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
         @Override public void init(long size, long updCntr, @Nullable Map<Integer, Long> cacheSizes) {
             initCntr = updCntr;
             storageSize.set(size);
+
             cntr.set(updCntr);
 
             if (cacheSizes != null) {

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/LocalJoinCachesContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/LocalJoinCachesContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/LocalJoinCachesContext.java
new file mode 100644
index 0000000..db829b2
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/LocalJoinCachesContext.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Context to capture caches state for a node joining to an active cluster. Since registered caches is updated in
+ * discovery thread and caches info are updated in exchange thread, we must capture the state in discovery thread
+ * somehow and pass it to the exchange. This class holds the required context.
+ */
+public class LocalJoinCachesContext {
+    /** */
+    @GridToStringInclude
+    private List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> locJoinStartCaches = Collections.emptyList();
+
+    /** */
+    @GridToStringInclude
+    private Map<Integer, CacheGroupDescriptor> cacheGrpDescs;
+
+    /** */
+    @GridToStringInclude
+    private Map<String, DynamicCacheDescriptor> cacheDescs;
+
+    /**
+     * @param locJoinStartCaches Local caches to start on join.
+     * @param cacheGrpDescs Cache group descriptors captured during join.
+     * @param cacheDescs Cache descriptors captured during join.
+     */
+    public LocalJoinCachesContext(
+        List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> locJoinStartCaches,
+        Map<Integer, CacheGroupDescriptor> cacheGrpDescs,
+        Map<String, DynamicCacheDescriptor> cacheDescs
+    ) {
+        this.locJoinStartCaches = locJoinStartCaches;
+        this.cacheGrpDescs = cacheGrpDescs;
+        this.cacheDescs = cacheDescs;
+    }
+
+    /**
+     * @return Caches to start.
+     */
+    public List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> caches() {
+        return locJoinStartCaches;
+    }
+
+    /**
+     * @return Group descriptors.
+     */
+    public Map<Integer, CacheGroupDescriptor> cacheGroupDescriptors() {
+        return cacheGrpDescs;
+    }
+
+    /**
+     * @return Cache descriptors.
+     */
+    public Map<String, DynamicCacheDescriptor> cacheDescriptors() {
+        return cacheDescs;
+    }
+
+    /**
+     * @param cacheNames Survived caches to clean.
+     */
+    public void removeSurvivedCaches(Set<String> cacheNames) {
+        if (cacheDescs != null) {
+            for (String cacheName : cacheNames)
+                cacheDescs.remove(cacheName);
+        }
+
+        Iterator<T2<DynamicCacheDescriptor, NearCacheConfiguration>> it = locJoinStartCaches.iterator();
+
+        for (; it.hasNext();) {
+            T2<DynamicCacheDescriptor, NearCacheConfiguration> entry = it.next();
+
+            DynamicCacheDescriptor desc = entry.get1();
+
+            if (cacheNames.contains(desc.cacheName()))
+                it.remove();
+        }
+    }
+
+    /**
+     * @param cacheGrps Survived caches groups to clean.
+     */
+    public void removeSurvivedCacheGroups(Set<Integer> cacheGrps) {
+        if (cacheGrpDescs != null) {
+            for (Integer grpId : cacheGrps)
+                cacheGrpDescs.remove(grpId);
+        }
+    }
+
+    /**
+     * @return {@code True} if the context is empty.
+     */
+    public boolean isEmpty() {
+        return F.isEmpty(locJoinStartCaches) && F.isEmpty(cacheGrpDescs) && F.isEmpty(cacheDescs);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(LocalJoinCachesContext.class, this);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StateChangeRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StateChangeRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StateChangeRequest.java
index 2d35e81..30a42bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StateChangeRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StateChangeRequest.java
@@ -19,8 +19,12 @@ package org.apache.ignite.internal.processors.cache;
 
 import java.util.UUID;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
+import org.apache.ignite.internal.processors.cluster.BaselineTopologyHistoryItem;
 import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteUuid;
+import org.jetbrains.annotations.Nullable;
 
 /**
  *
@@ -30,6 +34,12 @@ public class StateChangeRequest {
     private final ChangeGlobalStateMessage msg;
 
     /** */
+    private final BaselineTopologyHistoryItem prevBltHistItem;
+
+    /** */
+    private final boolean activeChanged;
+
+    /** */
     private final AffinityTopologyVersion topVer;
 
     /**
@@ -37,8 +47,12 @@ public class StateChangeRequest {
      * @param topVer State change topology versoin.
      */
     public StateChangeRequest(ChangeGlobalStateMessage msg,
+        BaselineTopologyHistoryItem bltHistItem,
+        boolean activeChanged,
         AffinityTopologyVersion topVer) {
         this.msg = msg;
+        prevBltHistItem = bltHistItem;
+        this.activeChanged = activeChanged;
         this.topVer = topVer;
     }
 
@@ -50,6 +64,13 @@ public class StateChangeRequest {
     }
 
     /**
+     * @return State change message ID.
+     */
+    public IgniteUuid id() {
+        return msg.id();
+    }
+
+    /**
      * @return State change request ID.
      */
     public UUID requestId() {
@@ -64,6 +85,27 @@ public class StateChangeRequest {
     }
 
     /**
+     * @return {@code True} if active state was changed.
+     */
+    public boolean activeChanged() {
+        return activeChanged;
+    }
+
+    /**
+     * @return Previous baseline topology.
+     */
+    @Nullable public BaselineTopologyHistoryItem prevBaselineTopologyHistoryItem() {
+        return prevBltHistItem;
+    }
+
+    /**
+     * @return Baseline topology.
+     */
+    @Nullable public BaselineTopology baselineTopology() {
+        return msg.baselineTopology();
+    }
+
+    /**
      * @return Node initiated state change process.
      */
     public UUID initiatorNodeId() {

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java
index a58918b..662839c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java
@@ -58,8 +58,8 @@ class BinaryMetadataFileStore {
         final ConcurrentMap<Integer, BinaryMetadataHolder> metadataLocCache,
         final GridKernalContext ctx,
         final IgniteLogger log,
-        @Nullable final File binaryMetadataFileStoreDir) throws IgniteCheckedException {
-
+        @Nullable final File binaryMetadataFileStoreDir
+    ) throws IgniteCheckedException {
         this.metadataLocCache = metadataLocCache;
         this.ctx = ctx;
         this.log = log;


[07/17] ignite git commit: IEP-4 Baseline topology for persistent caches (Phase 1) Contributed by: Dmitriy Govorukhin Dmitry Pavlov Eduard Shangareev Ily

Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionKey.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionKey.java
new file mode 100644
index 0000000..7c9371e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionKey.java
@@ -0,0 +1,103 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Partition key - pair of cache group ID and partition ID.
+ */
+public class PartitionKey implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Group id. */
+    private final int grpId;
+
+    /** Group name. Optional field, used only for output. */
+    private volatile String grpName;
+
+    /** Partition id. */
+    private final int partId;
+
+    /**
+     * @param grpId Group id.
+     * @param partId Partition id.
+     * @param grpName Group name.
+     */
+    public PartitionKey(int grpId, int partId, String grpName) {
+        this.grpId = grpId;
+        this.partId = partId;
+        this.grpName = grpName;
+    }
+
+    /**
+     * @return Group id.
+     */
+    public int groupId() {
+        return grpId;
+    }
+
+    /**
+     * @return Partition id.
+     */
+    public int partitionId() {
+        return partId;
+    }
+
+    /**
+     * @return Group name.
+     */
+    public String groupName() {
+        return grpName;
+    }
+
+    /**
+     * @param grpName Group name.
+     */
+    public void groupName(String grpName) {
+        this.grpName = grpName;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean equals(Object o) {
+        if (this == o)
+            return true;
+
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        PartitionKey key = (PartitionKey)o;
+
+        return grpId == key.grpId && partId == key.partId;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int hashCode() {
+        int res = grpId;
+
+        res = 31 * res + partId;
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(PartitionKey.class, this);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/RetrieveConflictPartitionValuesTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/RetrieveConflictPartitionValuesTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/RetrieveConflictPartitionValuesTask.java
new file mode 100644
index 0000000..d2ce882
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/RetrieveConflictPartitionValuesTask.java
@@ -0,0 +1,204 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeJob;
+import org.apache.ignite.compute.ComputeJobAdapter;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.compute.ComputeJobResultPolicy;
+import org.apache.ignite.compute.ComputeTaskAdapter;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.CacheObjectUtils;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+@GridInternal
+public class RetrieveConflictPartitionValuesTask extends ComputeTaskAdapter<Map<PartitionHashRecord, List<PartitionEntryHashRecord>>,
+    Map<PartitionHashRecord, List<PartitionEntryHashRecord>>> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Injected logger. */
+    @LoggerResource
+    private IgniteLogger log;
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(
+        List<ClusterNode> subgrid,
+        Map<PartitionHashRecord, List<PartitionEntryHashRecord>> collectTaskRes
+    ) throws IgniteException {
+        Map<ComputeJob, ClusterNode> jobs = new HashMap<>();
+
+        Map<Object, ClusterNode> consIdToNode = new HashMap<>();
+
+        for (ClusterNode node : subgrid)
+            consIdToNode.put(node.consistentId(), node);
+
+        for (Map.Entry<PartitionHashRecord, List<PartitionEntryHashRecord>> e : collectTaskRes.entrySet())
+            jobs.put(new RetrieveConflictValuesJob(new T2<>(e.getKey(), e.getValue())), consIdToNode.get(e.getKey().consistentId()));
+
+        return jobs;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Map<PartitionHashRecord, List<PartitionEntryHashRecord>> reduce(List<ComputeJobResult> results)
+        throws IgniteException {
+        Map<PartitionHashRecord, List<PartitionEntryHashRecord>> totalRes = new HashMap<>();
+
+        for (ComputeJobResult res : results) {
+            T2<PartitionHashRecord, List<PartitionEntryHashRecord>> nodeRes = res.getData();
+
+            totalRes.put(nodeRes.get1(), nodeRes.get2());
+        }
+
+        return totalRes;
+    }
+
+    /** {@inheritDoc} */
+    @Override public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) {
+        ComputeJobResultPolicy superRes = super.result(res, rcvd);
+
+        // Deny failover.
+        if (superRes == ComputeJobResultPolicy.FAILOVER) {
+            superRes = ComputeJobResultPolicy.WAIT;
+
+            log.warning("RetrieveConflictValuesJob failed on node " +
+                "[consistentId=" + res.getNode().consistentId() + "]", res.getException());
+        }
+
+        return superRes;
+    }
+
+    /**
+     *
+     */
+    public static class RetrieveConflictValuesJob extends ComputeJobAdapter {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** Ignite instance. */
+        @IgniteInstanceResource
+        private IgniteEx ignite;
+
+        /** Injected logger. */
+        @LoggerResource
+        private IgniteLogger log;
+
+        /** Partition hash record. */
+        private PartitionHashRecord partHashRecord;
+
+        /** Entry hash records. */
+        private List<PartitionEntryHashRecord> entryHashRecords;
+
+        /** Partition key. */
+        private PartitionKey partKey;
+
+        /**
+         * @param arg Partition key.
+         */
+        private RetrieveConflictValuesJob(T2<PartitionHashRecord, List<PartitionEntryHashRecord>> arg) {
+            partHashRecord = arg.get1();
+            entryHashRecords = arg.get2();
+            partKey = partHashRecord.partitionKey();
+        }
+
+        /** {@inheritDoc} */
+        @Override public Map<PartitionHashRecord, List<PartitionEntryHashRecord>> execute() throws IgniteException {
+            CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(partKey.groupId());
+
+            if (grpCtx == null)
+                return Collections.emptyMap();
+
+            GridDhtLocalPartition part = grpCtx.topology().localPartition(partKey.partitionId());
+
+            if (part == null || !part.reserve())
+                return Collections.emptyMap();
+
+            HashMap<Integer, GridCacheContext> cacheIdToCtx = new HashMap<>();
+
+            for (GridCacheContext ctx : grpCtx.caches())
+                cacheIdToCtx.put(ctx.cacheId(), ctx);
+
+            try {
+                if (part.state() != GridDhtPartitionState.OWNING)
+                    return Collections.emptyMap();
+
+                if (part.updateCounter() != partHashRecord.updateCounter()) {
+                    throw new IgniteException("Cluster is not idle: update counter of partition " + partKey.toString() +
+                        " changed during hash calculation [before=" + partHashRecord.updateCounter() +
+                        ", after=" + part.updateCounter() + "]");
+                }
+
+                for (PartitionEntryHashRecord entryHashRecord : entryHashRecords) {
+                    GridCacheContext ctx = cacheIdToCtx.get(entryHashRecord.cacheId());
+
+                    if (ctx == null)
+                        continue;
+
+                    KeyCacheObject key = grpCtx.shared().kernalContext().cacheObjects().toKeyCacheObject(
+                        grpCtx.cacheObjectContext(), entryHashRecord.key().cacheObjectType(), entryHashRecord.keyBytes());
+
+                    CacheDataRow row = part.dataStore().find(ctx, key);
+
+                    if (row == null)
+                        continue;
+
+                    CacheObject val = row.value();
+
+                    Object o = CacheObjectUtils.unwrapBinaryIfNeeded(grpCtx.cacheObjectContext(), val, true, true);
+
+                    if (o != null)
+                        entryHashRecord.valueString(o.toString());
+
+                    entryHashRecord.valueBytes(row.value().valueBytes(grpCtx.cacheObjectContext()));
+                }
+            }
+            catch (IgniteCheckedException e) {
+                U.error(log, "Can't retrieve value for partition " + partKey.toString(), e);
+
+                return Collections.emptyMap();
+            }
+            finally {
+                part.release();
+            }
+
+            return new T2<>(partHashRecord, entryHashRecords);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
new file mode 100644
index 0000000..23aa0e1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
@@ -0,0 +1,280 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeJob;
+import org.apache.ignite.compute.ComputeJobAdapter;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.compute.ComputeJobResultPolicy;
+import org.apache.ignite.compute.ComputeTaskAdapter;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.lang.GridIterator;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Task for comparing update counters and checksums between primary and backup partitions of specified caches.
+ * <br>
+ * Argument: Set of cache names, 'null' will trigger verification for all caches.
+ * <br>
+ * Result: If there are any update counter conflicts (which signals about concurrent updates in
+ * cluster), map with all counter conflicts is returned. Otherwise, map with all hash conflicts is returned.
+ * Each conflict is represented by list of {@link PartitionHashRecord} with data from different nodes.
+ * Successful verification always returns empty map.
+ * <br>
+ * Works properly only on idle cluster - there may be false positive conflict reports if data in cluster is being
+ * concurrently updated.
+ */
+@GridInternal
+public class VerifyBackupPartitionsTask extends ComputeTaskAdapter<Set<String>,
+    Map<PartitionKey, List<PartitionHashRecord>>> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Injected logger. */
+    @LoggerResource
+    private IgniteLogger log;
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(
+        List<ClusterNode> subgrid, Set<String> cacheNames) throws IgniteException {
+        Map<ComputeJob, ClusterNode> jobs = new HashMap<>();
+
+        for (ClusterNode node : subgrid)
+            jobs.put(new VerifyBackupPartitionsJob(cacheNames), node);
+
+        return jobs;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Map<PartitionKey, List<PartitionHashRecord>> reduce(List<ComputeJobResult> results)
+        throws IgniteException {
+        Map<PartitionKey, List<PartitionHashRecord>> clusterHashes = new HashMap<>();
+
+        for (ComputeJobResult res : results) {
+            Map<PartitionKey, PartitionHashRecord> nodeHashes = res.getData();
+
+            for (Map.Entry<PartitionKey, PartitionHashRecord> e : nodeHashes.entrySet()) {
+                if (!clusterHashes.containsKey(e.getKey()))
+                    clusterHashes.put(e.getKey(), new ArrayList<PartitionHashRecord>());
+
+                clusterHashes.get(e.getKey()).add(e.getValue());
+            }
+        }
+
+        Map<PartitionKey, List<PartitionHashRecord>> hashConflicts = new HashMap<>();
+
+        Map<PartitionKey, List<PartitionHashRecord>> updateCntrConflicts = new HashMap<>();
+
+        for (Map.Entry<PartitionKey, List<PartitionHashRecord>> e : clusterHashes.entrySet()) {
+            Integer partHash = null;
+            Long updateCntr = null;
+
+            for (PartitionHashRecord record : e.getValue()) {
+                if (partHash == null) {
+                    partHash = record.partitionHash();
+
+                    updateCntr = record.updateCounter();
+                }
+                else {
+                    if (record.updateCounter() != updateCntr) {
+                        updateCntrConflicts.put(e.getKey(), e.getValue());
+
+                        break;
+                    }
+
+                    if (record.partitionHash() != partHash) {
+                        hashConflicts.put(e.getKey(), e.getValue());
+
+                        break;
+                    }
+                }
+            }
+        }
+
+        return updateCntrConflicts.isEmpty() ? hashConflicts : updateCntrConflicts;
+    }
+
+    /** {@inheritDoc} */
+    @Override public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) {
+        ComputeJobResultPolicy superRes = super.result(res, rcvd);
+
+        // Deny failover.
+        if (superRes == ComputeJobResultPolicy.FAILOVER) {
+            superRes = ComputeJobResultPolicy.WAIT;
+
+            log.warning("VerifyBackupPartitionsJob failed on node " +
+                "[consistentId=" + res.getNode().consistentId() + "]", res.getException());
+        }
+
+        return superRes;
+    }
+
+    /**
+     *
+     */
+    public static class VerifyBackupPartitionsJob extends ComputeJobAdapter {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** Ignite instance. */
+        @IgniteInstanceResource
+        private IgniteEx ignite;
+
+        /** Injected logger. */
+        @LoggerResource
+        private IgniteLogger log;
+
+        /** Cache names. */
+        private Set<String> cacheNames;
+
+        /**
+         * @param names Names.
+         */
+        private VerifyBackupPartitionsJob(Set<String> names) {
+            cacheNames = names;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Map<PartitionKey, PartitionHashRecord> execute() throws IgniteException {
+            Set<Integer> grpIds = new HashSet<>();
+
+            Set<String> missingCaches = new HashSet<>();
+
+            if (cacheNames != null) {
+                for (String cacheName : cacheNames) {
+                    DynamicCacheDescriptor desc = ignite.context().cache().cacheDescriptor(cacheName);
+
+                    if (desc == null) {
+                        missingCaches.add(cacheName);
+
+                        continue;
+                    }
+
+                    grpIds.add(desc.groupId());
+                }
+
+                if (!missingCaches.isEmpty()) {
+                    StringBuilder strBuilder = new StringBuilder("The following caches do not exist: ");
+
+                    for (String name : missingCaches)
+                        strBuilder.append(name).append(", ");
+
+                    strBuilder.delete(strBuilder.length() - 2, strBuilder.length());
+
+                    throw new IgniteException(strBuilder.toString());
+                }
+            }
+            else {
+                Collection<CacheGroupContext> groups = ignite.context().cache().cacheGroups();
+
+                for (CacheGroupContext grp : groups) {
+                    if (!grp.systemCache() && !grp.isLocal())
+                        grpIds.add(grp.groupId());
+                }
+            }
+
+            Map<PartitionKey, PartitionHashRecord> res = new HashMap<>();
+
+            for (Integer grpId : grpIds) {
+                CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
+
+                if (grpCtx == null)
+                    continue;
+
+                List<GridDhtLocalPartition> parts = grpCtx.topology().localPartitions();
+
+                for (GridDhtLocalPartition part : parts) {
+                    if (!part.reserve())
+                        continue;
+
+                    int partHash = 0;
+                    long partSize;
+                    long updateCntrBefore;
+
+                    try {
+                        if (part.state() != GridDhtPartitionState.OWNING)
+                            continue;
+
+                        updateCntrBefore = part.updateCounter();
+
+                        partSize = part.dataStore().fullSize();
+
+                        GridIterator<CacheDataRow> it = grpCtx.offheap().partitionIterator(part.id());
+
+                        while (it.hasNextX()) {
+                            CacheDataRow row = it.nextX();
+
+                            partHash += row.key().hashCode();
+
+                            partHash += Arrays.hashCode(row.value().valueBytes(grpCtx.cacheObjectContext()));
+                        }
+
+                        long updateCntrAfter = part.updateCounter();
+
+                        if (updateCntrBefore != updateCntrAfter) {
+                            throw new IgniteException("Cluster is not idle: update counter of partition [grpId=" +
+                                grpId + ", partId=" + part.id() + "] changed during hash calculation [before=" +
+                                updateCntrBefore + ", after=" + updateCntrAfter + "]");
+                        }
+                    }
+                    catch (IgniteCheckedException e) {
+                        U.error(log, "Can't calculate partition hash [grpId=" + grpId +
+                            ", partId=" + part.id() + "]", e);
+
+                        continue;
+                    }
+                    finally {
+                        part.release();
+                    }
+
+                    Object consId = ignite.context().discovery().localNode().consistentId();
+
+                    boolean isPrimary = part.primary(grpCtx.topology().readyTopologyVersion());
+
+                    PartitionKey partKey = new PartitionKey(grpId, part.id(), grpCtx.cacheOrGroupName());
+
+                    res.put(partKey, new PartitionHashRecord(
+                        partKey, isPrimary, consId, partHash, updateCntrBefore, partSize));
+                }
+            }
+
+            return res;
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopology.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopology.java
new file mode 100644
index 0000000..fcb7a39
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopology.java
@@ -0,0 +1,484 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.processors.cluster;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.cluster.DetachedClusterNode;
+import org.apache.ignite.internal.cluster.NodeOrderComparator;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * BaselineTopology represents a set of "database nodes" - nodes responsible for holding persistent-enabled caches
+ * and persisting their information to durable storage.
+ *
+ * Two major features BaselineTopology allows are:
+ * <ol>
+ *     <li>Protection from conflicting updates.</li>
+ *     <li>Cluster auto activation.</li>
+ * </ol>
+ *
+ * <h2>Protection from conflicting updates</h2>
+ * <p>
+ *     Consider en example: there is a cluster of three nodes, A, B and C, all nodes store persistent data.
+ *
+ *     Cluster history looks like this:
+ *           [A,B,C]
+ *           |     \
+ *      (1)cluster segmentation, two parts get activated independently
+ *           |     |
+ *         [A,B]  [C]
+ *           |     |
+ *      (2)updates to both parts
+ *
+ *      After independent updates applied to both parts of cluster at point(2) node C should not be allowed to join
+ *      [A,B] part.
+ *
+ *      The following algorithm makes sure node C will never join [A,B] part back:
+ *      <ol>
+ *          <li>
+ *              When the cluster is initially activated BaselineTopology is created; its property <b>branchingPntHash</b>
+ *              is calculated based on consistent IDs of all nodes.
+ *          </li>
+ *          <li>
+ *              At point(1) two parts are activated separately; BaselineTopology in each is updated:
+ *              old <b>branchingPntHash</b> is moved to <b>branchingHistory</b> list;
+ *              new one is calculated based on new set of nodes.
+ *          </li>
+ *          <li>
+ *              If node C tries to join other part of cluster (e.g. after network connectivity repair)
+ *              [A,B] cluster checks its <b>branchingPntHash</b> and <b>branchingHistory</b>
+ *              to see if branching history of C's BaselineTopology has diverged from [A,B].
+ *              If divergence is detected [A,B] cluster refuses to let C into topology.
+ *          </li>
+ *      </ol>
+ *
+ *      All new nodes joining a cluster should pass validation process before join; detailed algorithm of the process
+ *      is described below.
+ * </p>
+ *
+ * <h3>Joining node validation algorithm</h3>
+ * Node joining a cluster firstly reads its local BaselineTopology from metastore and sends it to the cluster.
+ *
+ * <ol>
+ *     <li>
+ *         When BaselineTopology is set (e.g. on first activation) or recreated (e.g. with set-baseline command)
+ *         its ID on all nodes is incremented by one.
+ *
+ *         So when cluster receives a join request with BaselineTopology it firstly compares joining node BlT ID with
+ *         local BlT ID.
+ *
+ *         If joining node has a BaselineTopology with ID greater than one in cluster it means that BlT was changed
+ *         more times there; therefore new node is not allowed to join the cluster.
+ *     </li>
+ *     <li>
+ *         If user manually activates cluster when some of Baseline nodes are offline no new BlT is created.
+ *         Instead current set of online nodes from BaselineTopology is used to update {@link BaselineTopology#branchingPntHash}
+ *         property of current BaselineTopology.
+ *         Old value of the property is moved to {@link BaselineTopology#branchingHist} list.
+ *
+ *         If joining node and local BlT IDs are the same then cluster takes <b>branchingPntHash</b> of joining node
+ *         and verifies that its local <b>branchingHist</b> contains that hash.
+ *
+ *         If joining node hash is not presented in cluster branching history list
+ *         it means that joining node was activated independently of currently running cluster;
+ *         therefore new node is not allowed to join the cluster.
+ *
+ *         If joining node hash is presented in the history, that it is safe to let the node join the cluster.
+ *     </li>
+ *     <li>
+ *         When BaselineTopology is recreated (e.g. with set-baseline command) previous BaselineTopology is moved
+ *         to BaselineHistory (consult source code of {@link GridClusterStateProcessor} for more details).
+ *
+ *         If cluster sees that joining node BlT ID is less than cluster BlT ID it looks up for BaselineHistory item
+ *         for new node ID.
+ *         Having this BaselineHistory item cluster verifies that branching history of the item contains
+ *         branching point hash of joining node
+ *         (similar check as in the case above with only difference that joining node BlT is compared against
+ *         BaselineHistory item instead of BaselineTopology).
+ *
+ *         If new node branching point hash is found in the history than node is allowed to join;
+ *         otherwise it is rejected.
+ *     </li>
+ * </ol>
+ *
+ * <h2>Cluster auto activation</h2>
+ */
+public class BaselineTopology implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Consistent ID comparator. */
+    private static final Comparator<Object> CONSISTENT_ID_COMPARATOR = new Comparator<Object>() {
+        @Override public int compare(Object o1, Object o2) {
+            if (o1 instanceof Comparable && o2 instanceof Comparable && o1.getClass().equals(o2.getClass()))
+                return ((Comparable)o1).compareTo(o2);
+
+            return o1.toString().compareTo(o2.toString());
+        }
+    };
+
+    /** */
+    private final int id;
+
+    /** Key - node consistent ID, value - node attribute map. */
+    private final Map<Object, Map<String, Object>> nodeMap;
+
+    /** Compact ID to consistent ID mapping. */
+    private final Map<Short, Object> compactIdMapping;
+
+    /** Consistent ID to compact ID mapping. */
+    private final Map<Object, Short> consistentIdMapping;
+
+    /** */
+    private BranchingPointType lastBranchingPointType;
+
+    /** Branching point hash. */
+    private long branchingPntHash;
+
+    /** History of branching events.
+     * Each time when branching point hash changes previous value is added to this list. */
+    private final List<Long> branchingHist;
+
+    /**
+     * @param nodeMap Map of node consistent ID to it's attributes.
+     */
+    private BaselineTopology(Map<Object, Map<String, Object>> nodeMap, int id) {
+        this.id = id;
+
+        compactIdMapping = U.newHashMap(nodeMap.size());
+        consistentIdMapping = U.newHashMap(nodeMap.size());
+
+        this.nodeMap = nodeMap;
+
+        Set<Object> consistentIds = new TreeSet<>(CONSISTENT_ID_COMPARATOR);
+
+        for (Object o : nodeMap.keySet()){
+            branchingPntHash += (long)o.hashCode();
+
+            consistentIds.add(o);
+        }
+
+        short compactId = 0;
+
+        for (Object consistentId : consistentIds) {
+            compactIdMapping.put(compactId, consistentId);
+
+            consistentIdMapping.put(consistentId, compactId++);
+        }
+
+        lastBranchingPointType = BranchingPointType.NEW_BASELINE_TOPOLOGY;
+
+        branchingHist = new ArrayList<>();
+
+        branchingHist.add(branchingPntHash);
+    }
+
+    /**
+     * @return id of this BaselineTopology.
+     */
+    public int id() {
+        return id;
+    }
+
+    /**
+     * @return Set of consistent IDs.
+     */
+    public Set<Object> consistentIds() {
+        return nodeMap.keySet();
+    }
+
+    /**
+     * @return Activation history.
+     */
+    public List<Long> branchingHistory() {
+        return branchingHist;
+    }
+
+    /**
+     * @return Compact IDs mapping.
+     */
+    public Map<Short, Object> compactIdMapping() {
+        return compactIdMapping;
+    }
+
+    /**
+     * @return Consistent IDs mapping.
+     */
+    public Map<Object, Short> consistentIdMapping() {
+        return consistentIdMapping;
+    }
+
+    /**
+     * @return Short consistent Id.
+     */
+    public Short resolveShortConsistentId(Object constId){
+        return consistentIdMapping.get(constId);
+    }
+
+    /**
+     * @return Object consistent Id.
+     */
+    public Object resolveConsistentId(Short constId){
+        return compactIdMapping.get(constId);
+    }
+
+
+    /**
+     * @return Activation hash.
+     */
+    public long branchingPointHash() {
+        return branchingPntHash;
+    }
+
+    /**
+     * @param consId Node consistent ID.
+     * @return Node attributes map.
+     */
+    public Map<String, Object> attributes(Object consId) {
+        return nodeMap.get(consId);
+    }
+
+    /**
+     *
+     */
+    public List<BaselineNode> currentBaseline() {
+        List<BaselineNode> res = new ArrayList<>();
+
+        for (Map.Entry<Object, Map<String, Object>> consIdAttrsEntry : nodeMap.entrySet())
+            res.add(new DetachedClusterNode(consIdAttrsEntry.getKey(), consIdAttrsEntry.getValue()));
+
+        return res;
+    }
+
+    /**
+     * @param consId Consistent ID.
+     * @return Baseline node, if present in the baseline, or {@code null} if absent.
+     */
+    public ClusterNode baselineNode(Object consId) {
+        Map<String, Object> attrs = nodeMap.get(consId);
+
+        return attrs != null ? new DetachedClusterNode(consId, attrs) : null;
+    }
+
+    /**
+     * @param aliveNodes Sorted list of currently alive nodes.
+     * @param nodeFilter Node filter.
+     * @return Sorted list of baseline topology nodes.
+     */
+    public List<ClusterNode> createBaselineView(
+        List<ClusterNode> aliveNodes,
+        @Nullable IgnitePredicate<ClusterNode> nodeFilter)
+    {
+        List<ClusterNode> res = new ArrayList<>(nodeMap.size());
+
+        for (ClusterNode node : aliveNodes) {
+            if (nodeMap.containsKey(node.consistentId()) && (nodeFilter == null || CU.affinityNode(node, nodeFilter)))
+                res.add(node);
+        }
+
+        assert res.size() <= nodeMap.size();
+
+        if (res.size() == nodeMap.size())
+            return res;
+
+        Map<Object, ClusterNode> consIdMap = new HashMap<>();
+
+        for (ClusterNode node : aliveNodes) {
+            if (nodeMap.containsKey(node.consistentId()) && (nodeFilter == null || CU.affinityNode(node, nodeFilter)))
+                consIdMap.put(node.consistentId(), node);
+        }
+
+        for (Map.Entry<Object, Map<String, Object>> e : nodeMap.entrySet()) {
+            Object consId = e.getKey();
+
+            if (!consIdMap.containsKey(consId)) {
+                DetachedClusterNode node = new DetachedClusterNode(consId, e.getValue());
+
+                if (nodeFilter == null || CU.affinityNode(node, nodeFilter))
+                    consIdMap.put(consId, node);
+            }
+        }
+
+        res = new ArrayList<>();
+
+        res.addAll(consIdMap.values());
+
+        Collections.sort(res, NodeOrderComparator.getInstance());
+
+        return res;
+    }
+
+    /**
+     * @param presentedNodes Nodes present in cluster.
+     * @return {@code True} if current topology satisfies baseline.
+     */
+    public boolean isSatisfied(@NotNull Collection<ClusterNode> presentedNodes) {
+        if (presentedNodes.size() < nodeMap.size())
+            return false;
+
+        Set<Object> presentedNodeIds = new HashSet<>();
+
+        for (ClusterNode node : presentedNodes)
+            presentedNodeIds.add(node.consistentId());
+
+        return presentedNodeIds.containsAll(nodeMap.keySet());
+    }
+
+    /**
+     * @return Size of the baseline topology.
+     */
+    public int size() {
+        return nodeMap.size();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        BaselineTopology top = (BaselineTopology)o;
+
+        return nodeMap != null ? nodeMap.keySet().equals(top.nodeMap.keySet()) : top.nodeMap == null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int hashCode() {
+        return nodeMap != null ? nodeMap.hashCode() : 0;
+    }
+
+    /**
+     * @param blt1 Baseline topology instance.
+     * @param blt2 Baseline topology instance.
+     * @return {@code True} if equals.
+     */
+    public static boolean equals(BaselineTopology blt1, BaselineTopology blt2) {
+        if (blt1 == null && blt2 == null)
+            return true;
+
+        if (blt1 == null ^ blt2 == null)
+            return false;
+
+        return blt1.equals(blt2);
+    }
+
+    /**
+     * @param nodes Nodes.
+     * @param id ID of BaselineTopology to build.
+     * @return Baseline topology consisting of given nodes.
+     */
+    @Nullable public static BaselineTopology build(Collection<? extends BaselineNode> nodes, int id) {
+        if (nodes == null)
+            return null;
+
+        Map<Object, Map<String, Object>> nodeMap = new HashMap<>();
+
+        for (BaselineNode node : nodes)
+            nodeMap.put(node.consistentId(), node.attributes());
+
+        return new BaselineTopology(nodeMap, id);
+    }
+
+    /**
+     * @param blt BaselineTopology to check.
+     * @return {@code True} if current BaselineTopology is compatible (the same or a newer one) with passed in Blt.
+     */
+    boolean isCompatibleWith(BaselineTopology blt) {
+        return blt == null || (branchingPntHash == blt.branchingPntHash) || branchingHist.contains(blt.branchingPntHash);
+    }
+
+    /**
+     * @param nodes Nodes.
+     */
+    boolean updateHistory(Collection<? extends BaselineNode> nodes) {
+        long newTopHash = calculateTopologyHash(nodes);
+
+        lastBranchingPointType = BranchingPointType.CLUSTER_ACTIVATION;
+
+        if (branchingPntHash != newTopHash) {
+            branchingPntHash = newTopHash;
+
+            branchingHist.add(newTopHash);
+
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * Resets branching history of the current BaselineTopology.
+     * Current branching point hash and list of previous branching hashes are erased
+     * and replaced with the hash passed via method parameter.
+     *
+     * All nodes that were offline when reset happened (thus didn't reset history of their local BaselineTopologies)
+     * won't be allowed to join the cluster when started back.
+     *
+     * @param newBranchingPointHash New branching point hash.
+     */
+    void resetBranchingHistory(long newBranchingPointHash) {
+        lastBranchingPointType = BranchingPointType.BRANCHING_HISTORY_RESET;
+
+        branchingHist.clear();
+
+        branchingPntHash = newBranchingPointHash;
+        branchingHist.add(newBranchingPointHash);
+    }
+
+    /**
+     * @param nodes Nodes.
+     */
+    private long calculateTopologyHash(Collection<? extends BaselineNode> nodes) {
+        long res = 0;
+
+        Set<Object> bltConsIds = nodeMap.keySet();
+
+        for (BaselineNode node : nodes) {
+            if (bltConsIds.contains(node.consistentId()))
+                res += (long) node.consistentId().hashCode();
+        }
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return "BaselineTopology [id=" + id
+            + ", branchingHash=" + branchingPntHash
+            + ", branchingType='" + lastBranchingPointType + '\''
+            + ", baselineNodes=" + nodeMap.keySet() + ']';
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopologyHistory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopologyHistory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopologyHistory.java
new file mode 100644
index 0000000..fdc6cbf
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopologyHistory.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cluster;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadOnlyMetastorage;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadWriteMetastorage;
+
+/**
+ *
+ */
+public class BaselineTopologyHistory implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private static final String METASTORE_BLT_HIST_PREFIX = "bltHist-";
+
+    /** */
+    private final Queue<BaselineTopologyHistoryItem> bufferedForStore =
+        new ConcurrentLinkedQueue<>();
+
+    /** */
+    private final List<BaselineTopologyHistoryItem> hist = new ArrayList<>();
+
+    /** */
+    void restoreHistory(ReadOnlyMetastorage metastorage, int lastId) throws IgniteCheckedException {
+        for (int i = 0; i < lastId; i++) {
+            BaselineTopologyHistoryItem histItem = (BaselineTopologyHistoryItem) metastorage.read(METASTORE_BLT_HIST_PREFIX + i);
+
+            if (histItem != null)
+                hist.add(histItem);
+            else
+                throw new IgniteCheckedException("Restoring of BaselineTopology history has failed, " +
+                    "expected history item not found for id=" + i
+                );
+        }
+    }
+
+    /** */
+    BaselineTopologyHistory tailFrom(int id) {
+        BaselineTopologyHistory tail = new BaselineTopologyHistory();
+
+        for (BaselineTopologyHistoryItem item : hist) {
+            if (item.id() >= id)
+                tail.hist.add(item);
+        }
+
+        return tail;
+    }
+
+    /** */
+    void writeHistoryItem(ReadWriteMetastorage metastorage, BaselineTopologyHistoryItem histItem)
+        throws IgniteCheckedException
+    {
+        if (histItem == null)
+            return;
+
+        hist.add(histItem);
+
+        metastorage.write(METASTORE_BLT_HIST_PREFIX + histItem.id(), histItem);
+    }
+
+    /** */
+    void removeHistory(ReadWriteMetastorage metastorage) throws IgniteCheckedException {
+        if (hist.isEmpty())
+            return;
+
+        for (BaselineTopologyHistoryItem histItem : hist)
+            metastorage.remove(METASTORE_BLT_HIST_PREFIX + histItem.id());
+
+        hist.clear();
+    }
+
+    /** */
+    boolean isCompatibleWith(BaselineTopology blt) {
+        BaselineTopologyHistoryItem histBlt = hist.get(blt.id());
+
+        return histBlt.branchingHistory().contains(blt.branchingPointHash());
+    }
+
+    /** */
+    boolean isEmpty() {
+        return hist.isEmpty();
+    }
+
+    /** */
+    void bufferHistoryItemForStore(BaselineTopologyHistoryItem histItem) {
+        hist.add(histItem);
+
+        bufferedForStore.add(histItem);
+    }
+
+    /** */
+    public List<BaselineTopologyHistoryItem> history() {
+        return Collections.unmodifiableList(hist);
+    }
+
+    /** */
+    void flushHistoryItems(ReadWriteMetastorage metastorage) throws IgniteCheckedException {
+        while(!bufferedForStore.isEmpty()) {
+            BaselineTopologyHistoryItem item = bufferedForStore.remove();
+
+            metastorage.write(METASTORE_BLT_HIST_PREFIX + item.id(), item);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopologyHistoryItem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopologyHistoryItem.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopologyHistoryItem.java
new file mode 100644
index 0000000..4b2f0b5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopologyHistoryItem.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cluster;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+public class BaselineTopologyHistoryItem implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private final int id;
+
+    /** */
+    private final Collection<Object> consIds;
+
+    /** */
+    private final List<Long> branchingHistory;
+
+    /**
+     * @param id Id.
+     * @param consIds Consistent IDs.
+     * @param branchingHistory Activation history.
+     */
+    private BaselineTopologyHistoryItem(int id, Collection<Object> consIds, List<Long> branchingHistory) {
+        this.id = id;
+        this.consIds = consIds;
+        this.branchingHistory = branchingHistory;
+    }
+
+    /**
+     * @param blt Baseline Topology.
+     */
+    public static BaselineTopologyHistoryItem fromBaseline(BaselineTopology blt) {
+        if (blt == null)
+            return null;
+
+        List<Long> fullActivationHistory = new ArrayList<>(blt.branchingHistory().size());
+
+        fullActivationHistory.addAll(blt.branchingHistory());
+
+        return new BaselineTopologyHistoryItem(blt.id(), U.arrayList(blt.consistentIds()), fullActivationHistory);
+    }
+
+    /**
+     * @return ID.
+     */
+    public int id() {
+        return id;
+    }
+
+    /**
+     *
+     */
+    public List<Long> branchingHistory() {
+        return branchingHistory;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BranchingPointType.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BranchingPointType.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BranchingPointType.java
new file mode 100644
index 0000000..97388df
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BranchingPointType.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cluster;
+
+/**
+ * Used only for logging purposes.
+ *
+ * Enables to show in logs what was the cause of the last change
+ * of {@link BaselineTopology#branchingPntHash branching point hash}.
+ */
+enum BranchingPointType {
+    /** */
+    CLUSTER_ACTIVATION("Cluster activation"),
+
+    /** */
+    NEW_BASELINE_TOPOLOGY("New BaselineTopology"),
+
+    /** */
+    BRANCHING_HISTORY_RESET("Branching history reset");
+
+    /** */
+    private String type;
+
+    /** */
+    BranchingPointType(String type) {
+        this.type = type;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return type;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java
index a1fbacf..d7dfa16 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java
@@ -34,23 +34,27 @@ public class ChangeGlobalStateFinishMessage implements DiscoveryCustomMessage {
     private static final long serialVersionUID = 0L;
 
     /** Custom message ID. */
-    private IgniteUuid id = IgniteUuid.randomUuid();
+    private final IgniteUuid id = IgniteUuid.randomUuid();
 
     /** State change request ID. */
-    private UUID reqId;
+    private final UUID reqId;
 
     /** New cluster state. */
-    private boolean clusterActive;
+    private final boolean clusterActive;
+
+    /** State change error. */
+    private Boolean transitionRes;
 
     /**
      * @param reqId State change request ID.
      * @param clusterActive New cluster state.
      */
-    public ChangeGlobalStateFinishMessage(UUID reqId, boolean clusterActive) {
+    public ChangeGlobalStateFinishMessage(UUID reqId, boolean clusterActive, Boolean transitionRes) {
         assert reqId != null;
 
         this.reqId = reqId;
         this.clusterActive = clusterActive;
+        this.transitionRes = transitionRes;
     }
 
     /**
@@ -67,6 +71,13 @@ public class ChangeGlobalStateFinishMessage implements DiscoveryCustomMessage {
         return clusterActive;
     }
 
+    /**
+     * @return Transition success status.
+     */
+    public boolean success() {
+        return transitionRes == null ? clusterActive : transitionRes;
+    }
+
     /** {@inheritDoc} */
     @Override public IgniteUuid id() {
         return id;

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java
index 6a642bc..50fc022 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java
@@ -53,6 +53,15 @@ public class ChangeGlobalStateMessage implements DiscoveryCustomMessage {
     private List<StoredCacheData> storedCfgs;
 
     /** */
+    @Nullable private BaselineTopology baselineTopology;
+
+    /** */
+    private boolean forceChangeBaselineTopology;
+
+    /** */
+    private long timestamp;
+
+    /** */
     @GridToStringExclude
     private transient ExchangeActions exchangeActions;
 
@@ -66,7 +75,10 @@ public class ChangeGlobalStateMessage implements DiscoveryCustomMessage {
         UUID reqId,
         UUID initiatingNodeId,
         @Nullable List<StoredCacheData> storedCfgs,
-        boolean activate
+        boolean activate,
+        BaselineTopology baselineTopology,
+        boolean forceChangeBaselineTopology,
+        long timestamp
     ) {
         assert reqId != null;
         assert initiatingNodeId != null;
@@ -75,6 +87,9 @@ public class ChangeGlobalStateMessage implements DiscoveryCustomMessage {
         this.initiatingNodeId = initiatingNodeId;
         this.storedCfgs = storedCfgs;
         this.activate = activate;
+        this.baselineTopology = baselineTopology;
+        this.forceChangeBaselineTopology = forceChangeBaselineTopology;
+        this.timestamp = timestamp;
     }
 
     /**
@@ -136,6 +151,27 @@ public class ChangeGlobalStateMessage implements DiscoveryCustomMessage {
     }
 
     /**
+     * @return Force change BaselineTopology flag.
+     */
+    public boolean forceChangeBaselineTopology() {
+        return forceChangeBaselineTopology;
+    }
+
+    /**
+     * @return Baseline topology.
+     */
+    @Nullable public BaselineTopology baselineTopology() {
+        return baselineTopology;
+    }
+
+    /**
+     * @return Timestamp.
+     */
+    public long timestamp() {
+        return timestamp;
+    }
+
+    /**
      * @return State change request ID.
      */
     public UUID requestId() {

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
index 71bf90b..1c8e830 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
@@ -38,6 +38,9 @@ public class DiscoveryDataClusterState implements Serializable {
     private final boolean active;
 
     /** */
+    @Nullable private final BaselineTopology baselineTopology;
+
+    /** */
     private final UUID transitionReqId;
 
     /** Topology version for state change exchange. */
@@ -51,12 +54,18 @@ public class DiscoveryDataClusterState implements Serializable {
     /** Local flag for state transition result (global state is updated asynchronously by custom message). */
     private transient volatile Boolean transitionRes;
 
+    /** */
+    private transient DiscoveryDataClusterState prevState;
+
+    /** */
+    private transient volatile Exception transitionError;
+
     /**
      * @param active Current status.
      * @return State instance.
      */
-    static DiscoveryDataClusterState createState(boolean active) {
-        return new DiscoveryDataClusterState(active, null, null, null);
+    static DiscoveryDataClusterState createState(boolean active, @Nullable BaselineTopology baselineTopology) {
+        return new DiscoveryDataClusterState(null, active, baselineTopology, null, null, null);
     }
 
     /**
@@ -66,28 +75,45 @@ public class DiscoveryDataClusterState implements Serializable {
      * @param transitionNodes Nodes participating in state change exchange.
      * @return State instance.
      */
-    static DiscoveryDataClusterState createTransitionState(boolean active,
+    static DiscoveryDataClusterState createTransitionState(
+        DiscoveryDataClusterState prevState,
+        boolean active,
+        @Nullable BaselineTopology baselineTopology,
         UUID transitionReqId,
         AffinityTopologyVersion transitionTopVer,
-        Set<UUID> transitionNodes) {
+        Set<UUID> transitionNodes
+    ) {
         assert transitionReqId != null;
         assert transitionTopVer != null;
         assert !F.isEmpty(transitionNodes) : transitionNodes;
 
-        return new DiscoveryDataClusterState(active, transitionReqId, transitionTopVer, transitionNodes);
+        return new DiscoveryDataClusterState(
+            prevState,
+            active,
+            baselineTopology,
+            transitionReqId,
+            transitionTopVer,
+            transitionNodes);
     }
 
     /**
+     * @param prevState Previous state. May be non-null only for transitional states.
      * @param active New state.
      * @param transitionReqId State change request ID.
      * @param transitionTopVer State change topology version.
      * @param transitionNodes Nodes participating in state change exchange.
      */
-    private DiscoveryDataClusterState(boolean active,
+    private DiscoveryDataClusterState(
+        DiscoveryDataClusterState prevState,
+        boolean active,
+        @Nullable BaselineTopology baselineTopology,
         @Nullable UUID transitionReqId,
         @Nullable AffinityTopologyVersion transitionTopVer,
-        @Nullable Set<UUID> transitionNodes) {
+        @Nullable Set<UUID> transitionNodes
+    ) {
+        this.prevState = prevState;
         this.active = active;
+        this.baselineTopology = baselineTopology;
         this.transitionReqId = transitionReqId;
         this.transitionTopVer = transitionTopVer;
         this.transitionNodes = transitionNodes;
@@ -108,11 +134,8 @@ public class DiscoveryDataClusterState implements Serializable {
      * @param active New cluster state.
      */
     public void setTransitionResult(UUID reqId, boolean active) {
-        if (reqId.equals(transitionReqId)) {
-            assert transitionRes == null : this;
-
+        if (reqId.equals(transitionReqId))
             transitionRes = active;
-        }
     }
 
     /**
@@ -144,12 +167,50 @@ public class DiscoveryDataClusterState implements Serializable {
     }
 
     /**
+     * @return Baseline topology.
+     */
+    @Nullable public BaselineTopology baselineTopology() {
+        return baselineTopology;
+    }
+
+    /**
      * @return Nodes participating in state change exchange.
      */
     public Set<UUID> transitionNodes() {
         return transitionNodes;
     }
 
+    /**
+     * @return Transition error.
+     */
+    @Nullable public Exception transitionError() {
+        return transitionError;
+    }
+
+    /**
+     * @param ex Exception
+     */
+    public void transitionError(Exception ex) {
+        transitionError = ex;
+    }
+
+    /**
+     * @param success Transition success status.
+     * @return Cluster state that finished transition.
+     */
+    public DiscoveryDataClusterState finish(boolean success) {
+        return success ?
+            new DiscoveryDataClusterState(
+                null,
+                active,
+                baselineTopology,
+                null,
+                null,
+                null
+            ) :
+            prevState;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(DiscoveryDataClusterState.class, this);


[03/17] ignite git commit: IEP-4 Baseline topology for persistent caches (Phase 1) Contributed by: Dmitriy Govorukhin Dmitry Pavlov Eduard Shangareev Ily

Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCacheRemoveFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCacheRemoveFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCacheRemoveFailoverTest.java
new file mode 100644
index 0000000..b3da12e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCacheRemoveFailoverTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractRemoveFailureTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
+/**
+ *
+ */
+public class IgniteStableBaselineCacheRemoveFailoverTest extends GridCacheAbstractRemoveFailureTest {
+    /** */
+    private static final int GRIDS_COUNT = 3;
+
+    /** */
+    private static final int OUT_OF_BASELINE_GRID_ID = GRIDS_COUNT;
+
+    /** {@inheritDoc} */
+    @Override protected CacheMode cacheMode() {
+        return PARTITIONED;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected CacheAtomicityMode atomicityMode() {
+        return TRANSACTIONAL;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected NearCacheConfiguration nearCache() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                    .setPersistenceEnabled(true)
+                    .setInitialSize(512 * 1024 * 1024)
+                    .setMaxSize(512 * 1024 * 1024)
+                    .setCheckpointPageBufferSize(512 * 1024 * 1024)
+                )
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        GridTestUtils.deleteDbFiles();
+
+        startGrids(GRIDS_COUNT);
+
+        grid(0).active(true);
+
+        startGrid(OUT_OF_BASELINE_GRID_ID);
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean stop, final AtomicReference<CyclicBarrier> cmp) {
+        return GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                Thread.currentThread().setName("restart-thread");
+
+                while (!stop.get()) {
+                    U.sleep(random(1000, 3000));
+
+                    if (ThreadLocalRandom.current().nextInt(5) < 3)
+                        killAndRestart(stop, random(1, GRIDS_COUNT + 1));
+                    else
+                        killAndRestart(stop, OUT_OF_BASELINE_GRID_ID);
+
+                    CyclicBarrier barrier = cmp.get();
+
+                    if (barrier != null) {
+                        log.info("Wait data check.");
+
+                        barrier.await(60_000, TimeUnit.MILLISECONDS);
+
+                        log.info("Finished wait data check.");
+                    }
+                }
+
+                return null;
+            }
+        });
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteCheckpointDirtyPagesForLowLoadTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteCheckpointDirtyPagesForLowLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteCheckpointDirtyPagesForLowLoadTest.java
new file mode 100644
index 0000000..34662a7
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteCheckpointDirtyPagesForLowLoadTest.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.checkpoint;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.LockSupport;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test generates low load to grid in having some shared groups. Test checks if pages marked dirty after some time will
+ * became reasonable low for 1 put.
+ */
+public class IgniteCheckpointDirtyPagesForLowLoadTest extends GridCommonAbstractTest {
+    /** Caches in group. */
+    private static final int CACHES_IN_GRP = 1;
+    /** Groups. */
+    private static final int GROUPS = 1;
+    /** Parts. */
+    private static final int PARTS = 1024;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        List<CacheConfiguration> ccfgs = new ArrayList<>();
+
+        for (int g = 0; g < GROUPS; g++) {
+            for (int i = 0; i < CACHES_IN_GRP; i++) {
+                CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>().setName("dummyCache" + i + "." + g)
+                    .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+                    .setGroupName("dummyGroup" + g)
+                    .setAffinity(new RendezvousAffinityFunction(false, PARTS));
+
+                ccfgs.add(ccfg);
+            }
+        }
+        cfg.setCacheConfiguration(ccfgs.toArray(new CacheConfiguration[ccfgs.size()]));
+
+        DataStorageConfiguration dsCfg = new DataStorageConfiguration();
+        dsCfg.setPageSize(1024); //smaller page to reduce overhead to short values
+        dsCfg.setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true));
+        dsCfg.setCheckpointFrequency(500);
+        dsCfg.setWalMode(WALMode.LOG_ONLY);
+        dsCfg.setWalHistorySize(1);
+
+        cfg.setDataStorageConfiguration(dsCfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+
+        deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+        deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "temp", false));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testManyCachesAndNotManyPuts() throws Exception {
+        try {
+            IgniteEx ignite = startGrid(0);
+            ignite.active(true);
+
+            log.info("Saving initial data to caches");
+
+            for (int g = 0; g < GROUPS; g++) {
+                for (int c = 0; c < CACHES_IN_GRP; c++) {
+                    ignite.cache("dummyCache" + c + "." + g)
+                        .putAll(new TreeMap<Long, Long>() {{
+                            for (int j = 0; j < PARTS; j++) {
+                                // to fill each partition cache with at least 1 element
+                                put((long)j, (long)j);
+                            }
+                        }});
+                }
+            }
+
+            GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)(ignite.context().cache().context().database());
+
+            Collection<Integer> pageCntObserved = new ArrayList<>();
+
+            boolean checkpointWithLowNumOfPagesFound = false;
+
+            for (int i = 0; i < 10; i++) {
+                Random random = new Random();
+                //touch some entry
+                int d = random.nextInt(PARTS) + PARTS;
+                int cIdx = random.nextInt(CACHES_IN_GRP);
+                int gIdx = random.nextInt(GROUPS);
+
+                String fullname = "dummyCache" + cIdx + "." + gIdx;
+
+                ignite.cache(fullname).put(d, d);
+
+                log.info("Put to " + fullname + " value " + d);
+
+                db.wakeupForCheckpoint("").get();
+
+                int currCpPages = waitForCurrentCheckpointPagesCounterUpdated(db);
+
+                pageCntObserved.add(currCpPages);
+
+                log.info("Current CP pages: " + currCpPages);
+
+                if (currCpPages < PARTS * GROUPS) {
+                    checkpointWithLowNumOfPagesFound = true;  //reasonable number of pages in CP
+                    break;
+                }
+            }
+
+            stopGrid(0);
+
+            assertTrue("All checkpoints mark too much pages: " + pageCntObserved,
+                checkpointWithLowNumOfPagesFound);
+
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * @param db DB shared manager.
+     * @return counter when it becomes non-zero.
+     */
+    private int waitForCurrentCheckpointPagesCounterUpdated(GridCacheDatabaseSharedManager db) {
+        int currCpPages = 0;
+
+        while (currCpPages == 0) {
+            LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1));
+            currCpPages = db.currentCheckpointPagesCount();
+        }
+
+        return currCpPages;
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
index 0dd9153..6f222f2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.processors.cache.persistence.db.file;
 
 import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -33,7 +34,6 @@ import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.nio.ByteOrder;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -47,35 +47,37 @@ import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.persistence.DummyPageIO;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
-import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.WALIterator;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
 import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.CacheObjectContext;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheOperation;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.persistence.DummyPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO;
+import org.apache.ignite.internal.util.lang.GridCloseableIterator;
+import org.apache.ignite.internal.util.lang.GridFilteredClosableIterator;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
@@ -269,11 +271,11 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
 
         wal = shared.wal();
 
-        try (WALIterator it = wal.replay(start)) {
-            it.nextX();
+        try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
+            it.next();
 
             for (FullPageId initialWrite : initWrites) {
-                IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
+                IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
 
                 assertTrue(String.valueOf(tup.get2()), tup.get2() instanceof PageSnapshot);
 
@@ -283,7 +285,7 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
 
                 //there are extra tracking pages, skip them
                 if (TrackingPageIO.VERSIONS.latest().trackingPageFor(actual.pageId(), mem.pageSize()) == actual.pageId()) {
-                    tup = it.nextX();
+                    tup = it.next();
 
                     assertTrue(tup.get2() instanceof PageSnapshot);
 
@@ -355,8 +357,8 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
 
         db.enableCheckpoints(false).get();
 
-        try (WALIterator it = wal.replay(start)) {
-            IgniteBiTuple<WALPointer, WALRecord> cpRecordTup = it.nextX();
+        try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
+            IgniteBiTuple<WALPointer, WALRecord> cpRecordTup = it.next();
 
             assert cpRecordTup.get2() instanceof CheckpointRecord;
 
@@ -372,7 +374,7 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
             CacheObjectContext coctx = cctx.cacheObjectContext();
 
             while (idx < entries.size()) {
-                IgniteBiTuple<WALPointer, WALRecord> dataRecTup = it.nextX();
+                IgniteBiTuple<WALPointer, WALRecord> dataRecTup = it.next();
 
                 assert dataRecTup.get2() instanceof DataRecord;
 
@@ -464,8 +466,8 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
 
         db.enableCheckpoints(false);
 
-        try (WALIterator it = wal.replay(start)) {
-            IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
+        try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
+            IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
 
             assert tup.get2() instanceof CheckpointRecord : tup.get2();
 
@@ -480,7 +482,7 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
             int idx = 0;
 
             while (idx < pageIds.size()) {
-                tup = it.nextX();
+                tup = it.next();
 
                 assert tup.get2() instanceof PageSnapshot : tup.get2().getClass();
 
@@ -488,7 +490,7 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
 
                 //there are extra tracking pages, skip them
                 if (TrackingPageIO.VERSIONS.latest().trackingPageFor(snap.fullPageId().pageId(), pageMem.pageSize()) == snap.fullPageId().pageId()) {
-                    tup = it.nextX();
+                    tup = it.next();
 
                     assertTrue(tup.get2() instanceof PageSnapshot);
 
@@ -656,15 +658,15 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
     ) throws Exception {
         Map<FullPageId, byte[]> replay = new HashMap<>();
 
-        try (WALIterator it = wal.replay(start)) {
-            IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
+        try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
+            IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
 
             assertTrue("Invalid record: " + tup, tup.get2() instanceof CheckpointRecord);
 
             CheckpointRecord cpRec = (CheckpointRecord)tup.get2();
 
-            while (it.hasNextX()) {
-                tup = it.nextX();
+            while (it.hasNext()) {
+                tup = it.next();
 
                 WALRecord rec = tup.get2();
 
@@ -1025,4 +1027,18 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
             mem.releasePage(fullId.groupId(), fullId.pageId(), page);
         }
     }
+
+    /**
+     *
+     */
+    private static class PartitionMetaStateRecordExcludeIterator extends GridFilteredClosableIterator<IgniteBiTuple<WALPointer, WALRecord>> {
+        private PartitionMetaStateRecordExcludeIterator(GridCloseableIterator<? extends IgniteBiTuple<WALPointer, WALRecord>> it) {
+            super(it);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected boolean accept(IgniteBiTuple<WALPointer, WALRecord> tup) {
+            return !(tup.get2() instanceof PartitionMetaStateRecord);
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
index 2bcc22f..2a3fc1f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.cache.persistence.db.wal;
 import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.nio.MappedByteBuffer;
 import java.nio.file.OpenOption;
 import org.apache.ignite.IgniteCache;
@@ -60,6 +61,9 @@ public class IgniteWalFlushFailoverTest extends GridCommonAbstractTest {
     /** */
     private boolean flushByTimeout;
 
+    /** */
+    private AtomicBoolean canFail = new AtomicBoolean();
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         deleteWorkFiles();
@@ -89,7 +93,7 @@ public class IgniteWalFlushFailoverTest extends GridCommonAbstractTest {
         DataStorageConfiguration memCfg = new DataStorageConfiguration()
                 .setDefaultDataRegionConfiguration(
                  new DataRegionConfiguration().setMaxSize(2048L * 1024 * 1024).setPersistenceEnabled(true))
-                .setFileIOFactory(new FailingFileIOFactory())
+                .setFileIOFactory(new FailingFileIOFactory(canFail))
                 .setWalMode(WALMode.BACKGROUND)
                 .setWalBufferSize(128 * 1024)// Setting WAL Segment size to high values forces flushing by timeout.
                 .setWalSegmentSize(flushByTimeout ? 500_000 : 50_000);
@@ -139,6 +143,8 @@ public class IgniteWalFlushFailoverTest extends GridCommonAbstractTest {
 
             final int iterations = 100;
 
+            canFail.set(true);
+
             for (int i = 0; i < iterations; i++) {
                 Transaction tx = grid.transactions().txStart(
                     TransactionConcurrency.PESSIMISTIC, TransactionIsolation.READ_COMMITTED);
@@ -177,8 +183,16 @@ public class IgniteWalFlushFailoverTest extends GridCommonAbstractTest {
         private static final long serialVersionUID = 0L;
 
         /** Delegate factory. */
+        private AtomicBoolean fail;
+
+        /** */
         private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory();
 
+        /** */
+        FailingFileIOFactory(AtomicBoolean fail) {
+            this.fail = fail;
+        }
+
         /** {@inheritDoc} */
         @Override public FileIO create(File file) throws IOException {
             return create(file, CREATE, READ, WRITE);
@@ -192,8 +206,9 @@ public class IgniteWalFlushFailoverTest extends GridCommonAbstractTest {
                 int writeAttempts = 2;
 
                 @Override public int write(ByteBuffer srcBuf) throws IOException {
-                    if (--writeAttempts == 0)
-                        throw new RuntimeException("Test exception. Unable to write to file.");
+
+                    if (--writeAttempts <= 0 && fail!= null && fail.get())
+                        throw new IOException("No space left on device");
 
                     return super.write(srcBuf);
                 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
index d6c0a9d..76c7851 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
@@ -87,7 +87,7 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
 
     /** {@inheritDoc} */
     @Override protected long getTestTimeout() {
-        return 30_000;
+        return 60_000;
     }
 
     /** {@inheritDoc} */
@@ -174,6 +174,8 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
 
                     startGrid(gridCount());
 
+                    grid.cluster().setBaselineTopology(grid.cluster().topologyVersion());
+
                     waitForRebalancing();
                 } catch (Exception expected) {
                     // There can be any exception. Do nothing.
@@ -194,6 +196,8 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
 
         stopAllGrids();
 
+        canFail.set(false);
+
         Ignite grid0 = startGrids(gridCount() + 1);
 
         grid0.active(true);
@@ -243,7 +247,7 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
                 int writeAttempts = 2;
 
                 @Override public int write(ByteBuffer srcBuf) throws IOException {
-                    if (--writeAttempts == 0 && fail!= null && fail.get())
+                    if (--writeAttempts <= 0 && fail!= null && fail.get())
                         throw new IOException("No space left on device");
 
                     return super.write(srcBuf);

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
index 80d49ed..7bb550d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
@@ -69,6 +69,7 @@ import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsConsistentIdProcessor;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
@@ -630,8 +631,6 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
         try {
             IgniteEx ctrlGrid = startGrid(0);
 
-            ctrlGrid.active(true);
-
             fork = true;
 
             IgniteEx cacheGrid = startGrid(1);
@@ -680,8 +679,6 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
         try {
             IgniteEx ctrlGrid = startGrid(0);
 
-            ctrlGrid.active(true);
-
             fork = true;
 
             IgniteEx cacheGrid = startGrid(1);
@@ -737,8 +734,6 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
         try {
             IgniteEx ctrlGrid = startGrid(0);
 
-            ctrlGrid.active(true);
-
             fork = true;
 
             IgniteEx cacheGrid = startGrid(1);
@@ -777,8 +772,6 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
         try {
             IgniteEx ctrlGrid = startGrid(0);
 
-            ctrlGrid.active(true);
-
             fork = true;
 
             IgniteEx cacheGrid = startGrid(1);
@@ -890,6 +883,279 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
     }
 
     /**
+     * @throws Exception If fail.
+     */
+    public void testMetastorage() throws Exception {
+        try {
+            int cnt = 5000;
+
+            IgniteEx ignite0 = (IgniteEx)startGrid("node1");
+            IgniteEx ignite1 = (IgniteEx)startGrid("node2");
+
+            ignite1.active(true);
+
+            GridCacheSharedContext<Object, Object> sharedCtx0 = ignite0.context().cache().context();
+            GridCacheSharedContext<Object, Object> sharedCtx1 = ignite1.context().cache().context();
+
+            MetaStorage storage0 = sharedCtx0.database().metaStorage();
+            MetaStorage storage1 = sharedCtx1.database().metaStorage();
+
+            assert storage0 != null;
+
+            for (int i = 0; i < cnt; i++) {
+                sharedCtx0.database().checkpointReadLock();
+
+                try {
+                    storage0.putData(String.valueOf(i), new byte[]{(byte)(i % 256), 2, 3});
+                }
+                finally {
+                    sharedCtx0.database().checkpointReadUnlock();
+                }
+
+                byte[] b1 = new byte[i + 3];
+                b1[0] = 1;
+                b1[1] = 2;
+                b1[2] = 3;
+
+                sharedCtx1.database().checkpointReadLock();
+
+                try {
+                    storage1.putData(String.valueOf(i), b1);
+                }
+                finally {
+                    sharedCtx1.database().checkpointReadUnlock();
+                }
+            }
+
+            for (int i = 0; i < cnt; i++) {
+                byte[] d1 = storage0.getData(String.valueOf(i));
+                assertEquals(3, d1.length);
+                assertEquals((byte)(i % 256), d1[0]);
+                assertEquals(2, d1[1]);
+                assertEquals(3, d1[2]);
+
+                byte[] d2 = storage1.getData(String.valueOf(i));
+                assertEquals(i + 3, d2.length);
+                assertEquals(1, d2[0]);
+                assertEquals(2, d2[1]);
+                assertEquals(3, d2[2]);
+            }
+
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * @throws Exception If fail.
+     */
+    public void testMetastorageLargeArray() throws Exception {
+        try {
+            int cnt = 5000;
+            int arraySize = 32_768;
+
+            IgniteEx ignite = (IgniteEx)startGrid("node1");
+
+            ignite.active(true);
+
+            GridCacheSharedContext<Object, Object> sharedCtx = ignite.context().cache().context();
+
+            MetaStorage storage = sharedCtx.database().metaStorage();
+
+            for (int i = 0; i < cnt; i++) {
+                byte[] b1 = new byte[arraySize];
+                for (int k = 0; k < arraySize; k++) {
+                    b1[k] = (byte) (k % 100);
+                }
+
+                sharedCtx.database().checkpointReadLock();
+
+                try {
+                    storage.putData(String.valueOf(i), b1);
+                }
+                finally {
+                    sharedCtx.database().checkpointReadUnlock();
+                }
+            }
+
+            for (int i = 0; i < cnt; i++) {
+                byte[] d2 = storage.getData(String.valueOf(i));
+                assertEquals(arraySize, d2.length);
+
+                for (int k = 0; k < arraySize; k++) {
+                    assertEquals((byte) (k % 100), d2[k]);
+                }
+            }
+
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * @throws Exception If fail.
+     */
+    public void testMetastorageRemove() throws Exception {
+        try {
+            int cnt = 400;
+
+            IgniteEx ignite0 = (IgniteEx)startGrid("node1");
+
+            ignite0.active(true);
+
+            GridCacheSharedContext<Object, Object> sharedCtx0 = ignite0.context().cache().context();
+
+            MetaStorage storage = sharedCtx0.database().metaStorage();
+
+            assert storage != null;
+
+            for (int i = 0; i < cnt; i++) {
+                sharedCtx0.database().checkpointReadLock();
+
+                try {
+                    storage.putData(String.valueOf(i), new byte[]{1, 2, 3});
+                }
+                finally {
+                    sharedCtx0.database().checkpointReadUnlock();
+                }
+            }
+
+            for (int i = 0; i < 10; i++) {
+                sharedCtx0.database().checkpointReadLock();
+
+                try {
+                    storage.removeData(String.valueOf(i));
+                }
+                finally {
+                    sharedCtx0.database().checkpointReadUnlock();
+                }
+            }
+
+            for (int i = 10; i < cnt; i++) {
+                byte[] d1 = storage.getData(String.valueOf(i));
+                assertEquals(3, d1.length);
+                assertEquals(1, d1[0]);
+                assertEquals(2, d1[1]);
+                assertEquals(3, d1[2]);
+            }
+
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * @throws Exception If fail.
+     */
+    public void testMetastorageUpdate() throws Exception {
+        try {
+            int cnt = 2000;
+
+            IgniteEx ignite0 = (IgniteEx)startGrid("node1");
+
+            ignite0.active(true);
+
+            GridCacheSharedContext<Object, Object> sharedCtx0 = ignite0.context().cache().context();
+
+            MetaStorage storage = sharedCtx0.database().metaStorage();
+
+            assert storage != null;
+
+            for (int i = 0; i < cnt; i++) {
+                sharedCtx0.database().checkpointReadLock();
+
+                try {
+                    storage.putData(String.valueOf(i), new byte[]{1, 2, 3});
+                }
+                finally {
+                    sharedCtx0.database().checkpointReadUnlock();
+                }
+            }
+
+            for (int i = 0; i < cnt; i++) {
+                sharedCtx0.database().checkpointReadLock();
+
+                try {
+                    storage.putData(String.valueOf(i), new byte[]{2, 2, 3, 4});
+                }
+                finally {
+                    sharedCtx0.database().checkpointReadUnlock();
+                }
+            }
+
+            for (int i = 0; i < cnt; i++) {
+                byte[] d1 = storage.getData(String.valueOf(i));
+                assertEquals(4, d1.length);
+                assertEquals(2, d1[0]);
+                assertEquals(2, d1[1]);
+                assertEquals(3, d1[2]);
+            }
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * @throws Exception If fail.
+     */
+    public void testMetastorageWalRestore() throws Exception {
+        try {
+            int cnt = 2000;
+
+            IgniteEx ignite0 = (IgniteEx)startGrid(0);
+
+            ignite0.active(true);
+
+            GridCacheSharedContext<Object, Object> sharedCtx0 = ignite0.context().cache().context();
+
+            MetaStorage storage = sharedCtx0.database().metaStorage();
+
+            assert storage != null;
+
+            for (int i = 0; i < cnt; i++) {
+                sharedCtx0.database().checkpointReadLock();
+
+                try {
+                    storage.putData(String.valueOf(i), new byte[]{1, 2, 3});
+                }
+                finally {
+                    sharedCtx0.database().checkpointReadUnlock();
+                }
+            }
+
+            for (int i = 0; i < cnt; i++) {
+                byte[] value = storage.getData(String.valueOf(i));
+                assert value != null;
+                assert value.length == 3;
+            }
+
+            stopGrid(0);
+
+            ignite0 = startGrid(0);
+
+            ignite0.active(true);
+
+            sharedCtx0 = ignite0.context().cache().context();
+
+            storage = sharedCtx0.database().metaStorage();
+
+            assert storage != null;
+
+            for (int i = 0; i < cnt; i++) {
+                byte[] value = storage.getData(String.valueOf(i));
+                assert value != null;
+            }
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
      * @throws Exception if failed.
      */
     public void testApplyDeltaRecords() throws Exception {
@@ -1576,6 +1842,8 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
 
                     return false;
                 }
+
+                assertTrue(Arrays.equals(data, (long[])val));
             }
 
             return true;

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalSerializerVersionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalSerializerVersionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalSerializerVersionTest.java
index 60fb1d0..fc61632 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalSerializerVersionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalSerializerVersionTest.java
@@ -23,20 +23,22 @@ import java.util.Iterator;
 import java.util.List;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.DataRegionConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.WALIterator;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
 import org.apache.ignite.internal.pagemem.wal.record.TimeStampRecord;
 import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer;
+import org.apache.ignite.internal.util.lang.GridCloseableIterator;
+import org.apache.ignite.internal.util.lang.GridFilteredClosableIterator;
 import org.apache.ignite.internal.util.typedef.internal.GPC;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
@@ -46,10 +48,10 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.apache.ignite.transactions.TransactionState;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
 import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
+import static org.apache.ignite.transactions.TransactionState.PREPARED;
 
 /**
  *
@@ -131,13 +133,7 @@ public class IgniteWalSerializerVersionTest extends GridCommonAbstractTest {
             @Override public List<WALRecord> call() throws Exception {
                 WALRecord rec0 = new DataRecord(Collections.<DataEntry>emptyList());
 
-                WALRecord rec1 = new TxRecord(
-                    TransactionState.PREPARED,
-                    null,
-                    null,
-                    null,
-                    null
-                );
+                WALRecord rec1 = new TxRecord(PREPARED,null,null,null);
 
                 return Arrays.asList(rec0, rec1);
             }
@@ -236,7 +232,7 @@ public class IgniteWalSerializerVersionTest extends GridCommonAbstractTest {
                 }
             }
             else
-                fail();
+                fail(String.valueOf(act));
         }
     }
 
@@ -273,7 +269,7 @@ public class IgniteWalSerializerVersionTest extends GridCommonAbstractTest {
 
         Iterator<Long> itToCheck = checker.getTimeStamps().iterator();
 
-        try (WALIterator it = wal.replay(p)) {
+        try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(p))) {
             while (it.hasNext()) {
                 IgniteBiTuple<WALPointer, WALRecord> tup0 = it.next();
 
@@ -293,6 +289,8 @@ public class IgniteWalSerializerVersionTest extends GridCommonAbstractTest {
         stopAllGrids();
 
         deleteWorkFiles();
+
+        System.clearProperty(IGNITE_WAL_SERIALIZER_VERSION);
     }
 
     /** {@inheritDoc} */
@@ -302,6 +300,8 @@ public class IgniteWalSerializerVersionTest extends GridCommonAbstractTest {
         stopAllGrids();
 
         deleteWorkFiles();
+
+        System.clearProperty(IGNITE_WAL_SERIALIZER_VERSION);
     }
 
     /** {@inheritDoc} */
@@ -315,4 +315,18 @@ public class IgniteWalSerializerVersionTest extends GridCommonAbstractTest {
     private void deleteWorkFiles() throws IgniteCheckedException {
         deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
     }
+
+    /**
+     *
+     */
+    private static class PartitionMetaStateRecordExcludeIterator extends GridFilteredClosableIterator<IgniteBiTuple<WALPointer, WALRecord>> {
+        private PartitionMetaStateRecordExcludeIterator(GridCloseableIterator<? extends IgniteBiTuple<WALPointer, WALRecord>> it) {
+            super(it);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected boolean accept(IgniteBiTuple<WALPointer, WALRecord> tup) {
+            return !(tup.get2() instanceof PartitionMetaStateRecord);
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
index 6b9f06a..d204347 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
@@ -254,7 +254,13 @@ public class WalCompactionTest extends GridCommonAbstractTest {
 
         File[] cpMarkers = cpMarkersDir.listFiles(new FilenameFilter() {
             @Override public boolean accept(File dir, String name) {
-                return !(name.equals(cpMarkersToSave[0].getName()) || name.equals(cpMarkersToSave[1].getName()));
+                return !(
+                    name.equals(cpMarkersToSave[0].getName()) ||
+                    name.equals(cpMarkersToSave[1].getName()) ||
+                    name.equals(cpMarkersToSave[2].getName()) ||
+                    name.equals(cpMarkersToSave[3].getName()) ||
+                    name.equals(cpMarkersToSave[4].getName())
+                );
             }
         });
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
index 10b6110..0b3d016 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
@@ -48,13 +48,14 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager;
 import org.apache.ignite.internal.processors.cache.IgniteRebalanceIterator;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
-import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeListImpl;
 import org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseListImpl;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -489,7 +490,7 @@ public class WalRecoveryTxLogicalRecordsTest extends GridCommonAbstractTest {
 
             File[] cpFiles = cpDir.listFiles();
 
-            assertTrue(cpFiles.length <= WAL_HIST_SIZE * 2);
+            assertTrue(cpFiles.length <= WAL_HIST_SIZE * 2 + 1); // starts & ends + node_start
         }
         finally {
             stopAllGrids();
@@ -897,14 +898,14 @@ public class WalRecoveryTxLogicalRecordsTest extends GridCommonAbstractTest {
         boolean foundTails = false;
 
         for (GridDhtLocalPartition part : parts) {
-            FreeListImpl freeList = GridTestUtils.getFieldValue(part.dataStore(), "freeList");
+            CacheFreeListImpl freeList = GridTestUtils.getFieldValue(part.dataStore(), "freeList");
 
             if (freeList == null)
                 // Lazy store.
                 continue;
 
             AtomicReferenceArray<PagesList.Stripe[]> buckets = GridTestUtils.getFieldValue(freeList,
-                FreeListImpl.class, "buckets");
+                AbstractFreeList.class, "buckets");
             //AtomicIntegerArray cnts = GridTestUtils.getFieldValue(freeList, PagesList.class, "cnts");
 
             assertNotNull(buckets);

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
index aa8d57f..63c219b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
@@ -1096,11 +1096,13 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
 
                             unwrappedKeyObj = key instanceof BinaryObject ? key : key.value(null, false);
                         }
-                        log.info("//Entry operation " + entry.op() + "; cache Id" + entry.cacheId() + "; " +
-                            "under transaction: " + globalTxId +
-                            //; entry " + entry +
-                            "; Key: " + unwrappedKeyObj +
-                            "; Value: " + unwrappedValObj);
+
+                        if (dumpRecords)
+                            log.info("//Entry operation " + entry.op() + "; cache Id" + entry.cacheId() + "; " +
+                                "under transaction: " + globalTxId +
+                                //; entry " + entry +
+                                "; Key: " + unwrappedKeyObj +
+                                "; Value: " + unwrappedValObj);
 
                         if (cacheObjHnd != null && (unwrappedKeyObj != null || unwrappedValObj != null))
                             cacheObjHnd.apply(unwrappedKeyObj, unwrappedValObj);
@@ -1113,8 +1115,9 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
                     final TxRecord txRecord = (TxRecord)walRecord;
                     final GridCacheVersion globalTxId = txRecord.nearXidVersion();
 
-                    log.info("//Tx Record, state: " + txRecord.state() +
-                        "; nearTxVersion" + globalTxId);
+                    if (dumpRecords)
+                        log.info("//Tx Record, state: " + txRecord.state() +
+                            "; nearTxVersion" + globalTxId);
                 }
             }
         }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/file/FileDownloaderTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/file/FileDownloaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/file/FileDownloaderTest.java
new file mode 100644
index 0000000..6d6e0c8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/file/FileDownloaderTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.File;
+import java.io.PrintWriter;
+import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import static org.junit.Assert.*;
+
+/**
+ * FileDownloader test
+ */
+public class FileDownloaderTest extends GridCommonAbstractTest {
+
+    private static final Path DOWNLOADER_PATH = new File("download").toPath();
+    private static final Path UPLOADER_PATH = new File("upload").toPath();
+
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        if (DOWNLOADER_PATH.toFile().exists())
+            DOWNLOADER_PATH.toFile().delete();
+
+        if (UPLOADER_PATH.toFile().exists())
+            UPLOADER_PATH.toFile().delete();
+    }
+
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        if (DOWNLOADER_PATH.toFile().exists())
+            DOWNLOADER_PATH.toFile().delete();
+
+        if (UPLOADER_PATH.toFile().exists())
+            UPLOADER_PATH.toFile().delete();
+    }
+
+    public void test() throws Exception {
+        //todo uncomment and fix
+       /* assertTrue(UPLOADER_PATH.toFile().createNewFile());
+        assertTrue(!DOWNLOADER_PATH.toFile().exists());
+
+        PrintWriter writer = new PrintWriter(UPLOADER_PATH.toFile());
+
+        for (int i = 0; i < 1_000_000; i++)
+            writer.write("HELLO WORLD");
+
+        writer.close();
+
+        FileDownloader downloader = new FileDownloader(DOWNLOADER_PATH, Executors.newSingleThreadExecutor());
+
+        InetSocketAddress address = downloader.start();
+
+        FileUploader uploader = new FileUploader(UPLOADER_PATH, address, Executors.newSingleThreadExecutor());
+
+        long size = uploader.upload().get();
+
+        downloader.download(size).get();
+
+        assertTrue(DOWNLOADER_PATH.toFile().exists());
+
+        assertEquals(UPLOADER_PATH.toFile().length(), DOWNLOADER_PATH.toFile().length());
+
+        assertArrayEquals(Files.readAllBytes(UPLOADER_PATH), Files.readAllBytes(DOWNLOADER_PATH));*/
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
new file mode 100644
index 0000000..4b5dbc9
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.internal.processors.database.IndexStorageSelfTest;
+import org.apache.ignite.internal.util.lang.GridInClosure3X;
+import org.apache.ignite.internal.util.typedef.CIX3;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.GridTestKernalContext;
+
+/**
+ *
+ */
+public class IndexStoragePageMemoryImplTest extends IndexStorageSelfTest {
+    /** Make sure page is small enough to trigger multiple pages in a linked list. */
+    public static final int PAGE_SIZE = 1024;
+
+    /** */
+    private static File allocationPath;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        allocationPath = U.resolveWorkDirectory(U.defaultWorkDirectory(), "pagemem", false);
+    }
+
+    /**
+     * @param clean Clean flag. If {@code true}, will clean previous memory state and allocate
+     *      new empty page memory.
+     * @return Page memory instance.
+     */
+    @Override protected PageMemory memory(boolean clean) throws Exception {
+        long[] sizes = new long[10];
+
+        for (int i = 0; i < sizes.length; i++)
+            sizes[i] = 1024 * 1024;
+
+        DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), allocationPath);
+
+        GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
+            new GridTestKernalContext(log),
+            null,
+            null,
+            null,
+            new NoOpPageStoreManager(),
+            new NoOpWALManager(),
+            new IgniteCacheDatabaseSharedManager(),
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null
+        );
+
+        return new PageMemoryImpl(
+            provider, sizes,
+            sharedCtx,
+            PAGE_SIZE,
+            new CIX3<FullPageId, ByteBuffer, Integer>() {
+                @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuf, Integer tag) {
+                    assert false : "No evictions should happen during the test";
+                }
+            },
+            new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
+                @Override public void applyx(Long page, FullPageId fullId, PageMemoryEx pageMem) {
+                }
+            }, new CheckpointLockStateChecker() {
+                @Override public boolean checkpointLockIsHeldByThread() {
+                    return true;
+                }
+            },
+            new DataRegionMetricsImpl(new DataRegionConfiguration()),
+            false
+        );
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java
deleted file mode 100644
index d5492ab..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence.pagemem;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-import org.apache.ignite.configuration.DataRegionConfiguration;
-import org.apache.ignite.internal.mem.DirectMemoryProvider;
-import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
-import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
-import org.apache.ignite.internal.processors.database.MetadataStorageSelfTest;
-import org.apache.ignite.internal.util.lang.GridInClosure3X;
-import org.apache.ignite.internal.util.typedef.CIX3;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.testframework.junits.GridTestKernalContext;
-
-/**
- *
- */
-public class MetadataStoragePageMemoryImplTest extends MetadataStorageSelfTest{
-    /** Make sure page is small enough to trigger multiple pages in a linked list. */
-    public static final int PAGE_SIZE = 1024;
-
-    /** */
-    private static File allocationPath;
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        allocationPath = U.resolveWorkDirectory(U.defaultWorkDirectory(), "pagemem", false);
-    }
-
-    /**
-     * @param clean Clean flag. If {@code true}, will clean previous memory state and allocate
-     *      new empty page memory.
-     * @return Page memory instance.
-     */
-    @Override protected PageMemory memory(boolean clean) throws Exception {
-        long[] sizes = new long[10];
-
-        for (int i = 0; i < sizes.length; i++)
-            sizes[i] = 1024 * 1024;
-
-        DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), allocationPath);
-
-        GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
-            new GridTestKernalContext(log),
-            null,
-            null,
-            null,
-            new NoOpPageStoreManager(),
-            new NoOpWALManager(),
-            new IgniteCacheDatabaseSharedManager(),
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null
-        );
-
-        return new PageMemoryImpl(
-            provider, sizes,
-            sharedCtx,
-            PAGE_SIZE,
-            new CIX3<FullPageId, ByteBuffer, Integer>() {
-                @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuf, Integer tag) {
-                    assert false : "No evictions should happen during the test";
-                }
-            },
-            new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
-                @Override public void applyx(Long page, FullPageId fullId, PageMemoryEx pageMem) {
-                }
-            }, new CheckpointLockStateChecker() {
-                @Override public boolean checkpointLockIsHeldByThread() {
-                    return true;
-                }
-            },
-            new DataRegionMetricsImpl(new DataRegionConfiguration()),
-            false
-        );
-    }
-}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
index 40887e8c..8f72ed6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
@@ -59,6 +59,11 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager {
     }
 
     /** {@inheritDoc} */
+    @Override public void initializeForMetastorage() throws IgniteCheckedException {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
     @Override public void shutdownForCacheGroup(CacheGroupContext grp, boolean destroy) throws IgniteCheckedException {
         // No-op.
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
index b95208c..00f3f8f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
@@ -72,7 +72,7 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager {
     }
 
     /** {@inheritDoc} */
-    @Override public int truncate(WALPointer ptr) {
+    @Override public int truncate(WALPointer low, WALPointer high) {
         return 0;
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
index 71107d4..71d23f2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
@@ -198,6 +198,7 @@ public abstract class IgniteChangeGlobalStateAbstractTest extends GridCommonAbst
         IgniteConfiguration cfg = getConfiguration(name);
 
         cfg.setConsistentId(node);
+        cfg.setAutoActivationEnabled(false);
 
         ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(backUpIpFinder);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java
index 02a21f4..1ef269e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java
@@ -127,7 +127,7 @@ public class IgniteChangeGlobalStateFailOverTest extends IgniteChangeGlobalState
                 }
             });
 
-            sleep(60_000);
+            sleep(30_000);
 
             stop.set(true);
 
@@ -255,7 +255,7 @@ public class IgniteChangeGlobalStateFailOverTest extends IgniteChangeGlobalState
                 }
             });
 
-            sleep(60_000);
+            sleep(30_000);
 
             stop.set(true);
 
@@ -362,7 +362,7 @@ public class IgniteChangeGlobalStateFailOverTest extends IgniteChangeGlobalState
                 }
             });
 
-            sleep(60_000);
+            sleep(30_000);
 
             stop.set(true);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
index 5552d70..0d9d9f2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
@@ -102,6 +102,8 @@ public abstract class IgniteAbstractStandByClientReconnectTest extends GridCommo
     @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(name);
 
+        cfg.setAutoActivationEnabled(false);
+
         if (!nodeClient.equals(name))
             cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(vmIpFinder));
         else {

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteStandByClientReconnectToNewClusterTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteStandByClientReconnectToNewClusterTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteStandByClientReconnectToNewClusterTest.java
index 2da32e3..871098f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteStandByClientReconnectToNewClusterTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteStandByClientReconnectToNewClusterTest.java
@@ -109,7 +109,7 @@ public class IgniteStandByClientReconnectToNewClusterTest extends IgniteAbstract
     /**
      * @throws Exception If failed.
      */
-    public void testActiveClientReconnectToInActiveCluster() throws Exception {
+    public void testActiveClientReconnectToInactiveCluster() throws Exception {
         startNodes(null);
 
         info(">>>> star grid");
@@ -189,7 +189,7 @@ public class IgniteStandByClientReconnectToNewClusterTest extends IgniteAbstract
     /**
      * @throws Exception If failed.
      */
-    public void testInActiveClientReconnectToActiveCluster() throws Exception {
+    public void testInactiveClientReconnectToActiveCluster() throws Exception {
         CountDownLatch activateLatch = new CountDownLatch(1);
 
         startNodes(activateLatch);
@@ -247,7 +247,7 @@ public class IgniteStandByClientReconnectToNewClusterTest extends IgniteAbstract
     /**
      * @throws Exception If failed.
      */
-    public void testInActiveClientReconnectToInActiveCluster() throws Exception {
+    public void testInActiveClientReconnectToInactiveCluster() throws Exception {
         startNodes(null);
 
         IgniteEx ig1 = grid(node1);


[16/17] ignite git commit: IEP-4 Baseline topology for persistent caches (Phase 1) Contributed by: Dmitriy Govorukhin Dmitry Pavlov Eduard Shangareev Ily

Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 8eaa003..cf331f8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -17,73 +17,643 @@
 
 package org.apache.ignite.internal.commandline;
 
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
 import org.apache.ignite.internal.client.GridClient;
+import org.apache.ignite.internal.client.GridClientAuthenticationException;
+import org.apache.ignite.internal.client.GridClientClosedException;
 import org.apache.ignite.internal.client.GridClientClusterState;
+import org.apache.ignite.internal.client.GridClientCompute;
 import org.apache.ignite.internal.client.GridClientConfiguration;
+import org.apache.ignite.internal.client.GridClientDisconnectedException;
 import org.apache.ignite.internal.client.GridClientException;
 import org.apache.ignite.internal.client.GridClientFactory;
+import org.apache.ignite.internal.client.GridClientHandshakeException;
+import org.apache.ignite.internal.client.GridClientNode;
+import org.apache.ignite.internal.client.GridServerUnreachableException;
+import org.apache.ignite.internal.client.impl.connection.GridClientConnectionResetException;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
+import org.apache.ignite.internal.visor.baseline.VisorBaselineNode;
+import org.apache.ignite.internal.visor.baseline.VisorBaselineOperation;
+import org.apache.ignite.internal.visor.baseline.VisorBaselineTask;
+import org.apache.ignite.internal.visor.baseline.VisorBaselineTaskArg;
+import org.apache.ignite.internal.visor.baseline.VisorBaselineTaskResult;
+
+import static org.apache.ignite.internal.IgniteVersionUtils.ACK_VER_STR;
+import static org.apache.ignite.internal.IgniteVersionUtils.COPYRIGHT;
+import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.ADD;
+import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.COLLECT;
+import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.REMOVE;
+import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.SET;
+import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.VERSION;
 
 /**
- *
+ * Class that execute several commands passed via command line.
  */
 public class CommandHandler {
+    /** */
+    private static final String DFLT_HOST = "127.0.0.1";
+
+    /** */
+    private static final String DFLT_PORT = "11211";
+
+    /** */
+    private static final String CMD_HELP = "--help";
+
+    /** */
+    private static final String CMD_HOST = "--host";
+
+    /** */
+    private static final String CMD_PORT = "--port";
+
+    /** */
+    private static final String CMD_ACTIVATE = "--activate";
+
+    /** */
+    private static final String CMD_DEACTIVATE = "--deactivate";
+
+    /** */
+    private static final String CMD_STATE = "--state";
+
+    /** */
+    private static final String CMD_BASE_LINE = "--baseline";
+
+    /** */
+    private static final String BASELINE_ADD = "add";
+
+    /** */
+    private static final String BASELINE_REMOVE = "remove";
+
+    /** */
+    private static final String BASELINE_SET = "set";
+
+    /** */
+    private static final String BASELINE_SET_VERSION = "version";
+
+    /** */
+    private static final String DELIM = "--------------------------------------------------------------------------------";
+
+    /** */
+    public static final int EXIT_CODE_OK = 0;
+
+    /** */
+    public static final int EXIT_CODE_INVALID_ARGUMENTS = 1;
+
+    /** */
+    public static final int EXIT_CODE_CONNECTION_FAILED = 2;
+
+    /** */
+    public static final int ERR_AUTHENTICATION_FAILED = 3;
+
+    /** */
+    public static final int EXIT_CODE_UNEXPECTED_ERROR = 4;
+
+    /**
+     * Output specified string to console.
+     *
+     * @param s String to output.
+     */
+    private void log(String s) {
+        System.out.println(s);
+    }
+
     /**
-     * @param args Args.
+     * Output empty line.
      */
-    public static void main(String[] args) throws GridClientException {
-        String host = "127.0.0.1";
-        String port = "11211";
-        Boolean activate = null;
+    private void nl() {
+        System.out.println("");
+    }
+
+    /**
+     * Print error to console.
+     *
+     * @param errCode Error code to return.
+     * @param s Optional message.
+     * @param e Error to print.
+     */
+    private int error(int errCode, String s, Throwable e) {
+        if (!F.isEmpty(s))
+            log(s);
 
-        if (args.length == 1 && "--help".equals(args[0])){
-            System.out.println("Example: --host {ip} --port {port} --{activate/deactivate} " +
-                "or without command --host {ip} --port {port} then will print status.");
+        String msg = e.getMessage();
 
-            return;
+        if (F.isEmpty(msg))
+            msg = e.getClass().getName();
+
+        if (msg.startsWith("Failed to handle request")) {
+            int p = msg.indexOf("err=");
+
+            msg = msg.substring(p + 4, msg.length() - 1);
         }
 
-        if (args.length > 5)
-            throw new IllegalArgumentException("incorrect number of arguments");
+        log("Error: " + msg);
+
+        return errCode;
+    }
+
+    /**
+     * Print command usage.
+     *
+     * @param desc Command description.
+     * @param cmd Command.
+     */
+    private void usage(String desc, String cmd) {
+        log(desc);
+        log("    control.sh [--host HOST_OR_IP] [--port PORT] " + cmd);
+        nl();
+    }
+
+    /**
+     * Extract next argument.
+     *
+     * @param it Arguments iterator.
+     * @param err Error message.
+     * @return Next argument value.
+     */
+    private String nextArg(Iterator<String> it, String err) {
+        if (it.hasNext()) {
+            String arg = it.next();
+
+            if (arg.startsWith("--"))
+                throw new IllegalArgumentException("Unexpected argument: " + arg);
+
+            return arg;
+        }
+
+        throw new IllegalArgumentException(err);
+    }
+
+    /**
+     * Activate cluster.
+     *
+     * @param client Client.
+     * @throws GridClientException If failed to activate.
+     */
+    private void activate(GridClient client) throws Throwable {
+        try {
+            GridClientClusterState state = client.state();
+
+            state.active(true);
 
-        for (int i = 0; i < args.length; i++) {
-            String str = args[i];
+            log("Cluster activated");
+        }
+        catch (Throwable e) {
+            log("Failed to activate cluster.");
 
-            if ("--host".equals(str))
-                host = args[i + 1];
-            else if ("--port".equals(str))
-                port = args[i + 1];
-            else if ("--activate".equals(str))
-                activate = true;
-            else if ("--deactivate".equals(str))
-                activate = false;
+            throw e;
         }
+    }
 
-        if (host == null)
-            throw new IllegalArgumentException("host can not be empty");
+    /**
+     * Deactivate cluster.
+     *
+     * @param client Client.
+     * @throws Throwable If failed to deactivate.
+     */
+    private void deactivate(GridClient client) throws Throwable {
+        try {
+            GridClientClusterState state = client.state();
 
-        if (port == null)
-            throw new IllegalArgumentException("port can not be empty");
+            state.active(false);
 
-        GridClientConfiguration cfg = new GridClientConfiguration();
-        cfg.setServers(Collections.singletonList(host + ":" + port));
+            log("Cluster deactivated");
+        }
+        catch (Throwable e) {
+            log("Failed to deactivate cluster.");
 
-        try (GridClient client = GridClientFactory.start(cfg)) {
+            throw e;
+        }
+    }
+
+    /**
+     * Print cluster state.
+     *
+     * @param client Client.
+     * @throws Throwable If failed to print state.
+     */
+    private void state(GridClient client) throws Throwable {
+        try {
             GridClientClusterState state = client.state();
 
-            if (activate != null)
+            log("Cluster is " + (state.active() ? "active" : "inactive"));
+        }
+        catch (Throwable e) {
+            log("Failed to get cluster state.");
+
+            throw e;
+        }
+    }
+
+    /**
+     *
+     * @param client Client
+     * @return Task result.
+     * @throws GridClientException If failed to execute task.
+     */
+    private <R> R executeTask(GridClient client, Class<?> taskCls, Object taskArgs) throws GridClientException {
+        GridClientCompute compute = client.compute();
+
+        List<GridClientNode> nodes = new ArrayList<>();
+
+        for (GridClientNode node : compute.nodes())
+            if (node.connectable())
+                nodes.add(node);
+
+        if (F.isEmpty(nodes))
+            throw new GridClientDisconnectedException("Connectable node not found", null);
+
+        GridClientNode node = compute.balancer().balancedNode(nodes);
+
+        return compute.projection(node).execute(taskCls.getName(),
+            new VisorTaskArgument<>(node.nodeId(), taskArgs, false));
+    }
+
+    /**
+     * Change baseline.
+     *
+     * @param client Client.
+     * @param baselineAct Baseline action to execute.  @throws GridClientException If failed to execute baseline action.
+     * @param baselineArgs Baseline action arguments.
+     * @throws Throwable If failed to execute baseline action.
+     */
+    private void baseline(GridClient client, String baselineAct, String baselineArgs) throws Throwable {
+        switch (baselineAct) {
+            case BASELINE_ADD:
+                baselineAdd(client, baselineArgs);
+                break;
+
+            case BASELINE_REMOVE:
+                baselineRemove(client, baselineArgs);
+                break;
+
+            case BASELINE_SET:
+                baselineSet(client, baselineArgs);
+                break;
+
+            case BASELINE_SET_VERSION:
+                baselineVersion(client, baselineArgs);
+                break;
+
+            default:
+                baselinePrint(client);
+        }
+    }
+
+    /**
+     * Prepare task argument.
+     *
+     * @param op Operation.
+     * @param s Argument from command line.
+     * @return Task argument.
+     */
+    private VisorBaselineTaskArg arg(VisorBaselineOperation op, String s) {
+        switch (op) {
+            case ADD:
+            case REMOVE:
+            case SET:
+                if(F.isEmpty(s))
+                    throw new IllegalArgumentException("Empty list of consistent IDs");
+
+                List<String> consistentIds = new ArrayList<>();
+
+                for (String consistentId : s.split(","))
+                    consistentIds.add(consistentId.trim());
+
+                return new VisorBaselineTaskArg(op, -1, consistentIds);
+
+            case VERSION:
                 try {
-                    state.active(activate);
+                    long topVer = Long.parseLong(s);
 
-                    System.out.println(host + ":" + port + " - was " + (activate ? "activate" : "deactivate"));
+                    return new VisorBaselineTaskArg(op, topVer, null);
                 }
-                catch (Exception e) {
-                    System.out.println("Something fail during " + (activate ? "activation" : "deactivation")
-                        + ", exception message: " + e.getMessage());
+                catch (NumberFormatException e) {
+                    throw new IllegalArgumentException("Invalid topology version: " + s, e);
                 }
-            else
-                System.out.println(host + ":" + port + " - " + (state.active() ? "active" : "inactive"));
 
+            default:
+                return new VisorBaselineTaskArg(op, -1, null);
         }
     }
+
+    /**
+     * Print baseline topology.
+     *
+     * @param res Task result with baseline topology.
+     */
+    private void baselinePrint0(VisorBaselineTaskResult res) {
+        log("Cluster state: " + (res.isActive() ? "active" : "inactive"));
+        log("Current topology version: " + res.getTopologyVersion());
+        nl();
+
+        Map<String, VisorBaselineNode> baseline = res.getBaseline();
+        Map<String, VisorBaselineNode> servers = res.getServers();
+
+        if (F.isEmpty(baseline))
+            log("Baseline nodes not found.");
+        else {
+            log("Baseline nodes:");
+
+            for(VisorBaselineNode node : baseline.values()) {
+                log("    ConsistentID=" + node.getConsistentId() + ", STATE=" +
+                    (servers.containsKey(node.getConsistentId()) ? "ONLINE" : "OFFLINE"));
+            }
+
+            log(DELIM);
+            log("Number of baseline nodes: " + baseline.size());
+
+            nl();
+
+            List<VisorBaselineNode> others = new ArrayList<>();
+
+            for (VisorBaselineNode node : servers.values()) {
+                if (!baseline.containsKey(node.getConsistentId()))
+                    others.add(node);
+            }
+
+            if (F.isEmpty(others))
+                log("Other nodes not found.");
+            else {
+                log("Other nodes:");
+
+                for(VisorBaselineNode node : others)
+                    log("    ConsistentID=" + node.getConsistentId());
+
+                log("Number of other nodes: " + others.size());
+            }
+        }
+    }
+
+    /**
+     * Print current baseline.
+     *
+     * @param client Client.
+     */
+    private void baselinePrint(GridClient client) throws GridClientException {
+        VisorBaselineTaskResult res = executeTask(client, VisorBaselineTask.class, arg(COLLECT, ""));
+
+        baselinePrint0(res);
+    }
+
+    /**
+     * Add nodes to baseline.
+     *
+     * @param client Client.
+     * @param baselineArgs Baseline action arguments.
+     * @throws Throwable If failed to add nodes to baseline.
+     */
+    private void baselineAdd(GridClient client, String baselineArgs) throws Throwable {
+        try {
+            VisorBaselineTaskResult res = executeTask(client, VisorBaselineTask.class, arg(ADD, baselineArgs));
+
+            baselinePrint0(res);
+        }
+        catch (Throwable e) {
+            log("Failed to add nodes to baseline.");
+
+            throw e;
+        }
+    }
+
+    /**
+     * Remove nodes from baseline.
+     *
+     * @param client Client.
+     * @param consistentIds Consistent IDs.
+     * @throws Throwable If failed to remove nodes from baseline.
+     */
+    private void baselineRemove(GridClient client, String consistentIds) throws Throwable {
+        try {
+            VisorBaselineTaskResult res = executeTask(client, VisorBaselineTask.class, arg(REMOVE, consistentIds));
+
+            baselinePrint0(res);
+        }
+        catch (Throwable e) {
+            log("Failed to remove nodes from baseline.");
+
+            throw e;
+        }
+    }
+
+    /**
+     * Set baseline.
+     *
+     * @param client Client.
+     * @param consistentIds Consistent IDs.
+     * @throws Throwable If failed to set baseline.
+     */
+    private void baselineSet(GridClient client, String consistentIds) throws Throwable {
+        try {
+            VisorBaselineTaskResult res = executeTask(client, VisorBaselineTask.class, arg(SET, consistentIds));
+
+            baselinePrint0(res);
+        }
+        catch (Throwable e) {
+            log("Failed to set baseline.");
+
+            throw e;
+        }
+    }
+
+    /**
+     * Set baseline by topology version.
+     *
+     * @param client Client.
+     * @param arg Argument from command line.
+     */
+    private void baselineVersion(GridClient client, String arg) throws GridClientException {
+        try {
+            VisorBaselineTaskResult res = executeTask(client, VisorBaselineTask.class, arg(VERSION, arg));
+
+            baselinePrint0(res);
+        }
+        catch (Throwable e) {
+            log("Failed to set baseline with specified topology version.");
+
+            throw e;
+        }
+    }
+
+    /**
+     * @param e Exception to check.
+     * @return {@code true} if specified exception is {@link GridClientAuthenticationException}.
+     */
+    private boolean isAuthError(Throwable e) {
+        return X.hasCause(e, GridClientAuthenticationException.class);
+    }
+
+    /**
+     * @param e Exception to check.
+     * @return {@code true} if specified exception is a connection error.
+     */
+    private boolean isConnectionError(Throwable e) {
+        return e instanceof GridClientClosedException ||
+            e instanceof GridClientConnectionResetException ||
+            e instanceof GridClientDisconnectedException ||
+            e instanceof GridClientHandshakeException ||
+            e instanceof GridServerUnreachableException;
+    }
+
+    /**
+     * Parse and execute command.
+     *
+     * @param args Arguments to parse and execute.
+     * @return Exit code.
+     */
+    public int execute(String... args) {
+        log("Control utility [ver. " + ACK_VER_STR + "]");
+        log(COPYRIGHT);
+        log("User: " + System.getProperty("user.name"));
+        log(DELIM);
+
+        try {
+            if (F.isEmpty(args) || (args.length == 1 && CMD_HELP.equalsIgnoreCase(args[0]))){
+                log("This utility can do the following commands:");
+
+                usage("  Activate cluster:", CMD_ACTIVATE);
+                usage("  Deactivate cluster:", CMD_DEACTIVATE);
+                usage("  Print current cluster state:", CMD_STATE);
+                usage("  Print cluster baseline topology:", CMD_BASE_LINE);
+                usage("  Add nodes into baseline topology:", CMD_BASE_LINE + " add consistentId1[,consistentId2,....,consistentIdN]");
+                usage("  Remove nodes from baseline topology:", CMD_BASE_LINE + " remove consistentId1[,consistentId2,....,consistentIdN]");
+                usage("  Set baseline topology:", CMD_BASE_LINE + " set consistentId1[,consistentId2,....,consistentIdN]");
+                usage("  Set baseline topology based on version:", CMD_BASE_LINE + " version topologyVersion");
+
+                log("Default values:");
+                log("    HOST_OR_IP=" + DFLT_HOST);
+                log("    PORT=" + DFLT_PORT);
+                nl();
+
+                log("Exit codes:");
+                log("    " + EXIT_CODE_OK + " - successful execution.");
+                log("    " + EXIT_CODE_INVALID_ARGUMENTS + " - invalid arguments.");
+                log("    " + EXIT_CODE_CONNECTION_FAILED + " - connection failed.");
+                log("    " + ERR_AUTHENTICATION_FAILED + " - authentication failed.");
+                log("    " + EXIT_CODE_UNEXPECTED_ERROR + " - unexpected error.");
+
+                return EXIT_CODE_OK;
+            }
+
+            String host = DFLT_HOST;
+
+            String port = DFLT_PORT;
+
+            String baselineAct = "";
+
+            String baselineArgs = "";
+
+            List<String> commands = new ArrayList<>();
+
+            Iterator<String> it = Arrays.asList(args).iterator();
+
+            while (it.hasNext()) {
+                String str = it.next().toLowerCase();
+
+                switch (str) {
+                    case CMD_HOST:
+                        host = nextArg(it, "Expected host name");
+                        break;
+
+                    case CMD_PORT:
+                        port = nextArg(it, "Expected port number");
+
+                        try {
+                            int p = Integer.parseInt(port);
+
+                            if (p <= 0 || p > 65535)
+                                throw new IllegalArgumentException("Invalid value for port: " + port);
+                        }
+                        catch (NumberFormatException ignored) {
+                            throw new IllegalArgumentException("Invalid value for port: " + port);
+                        }
+                        break;
+
+                    case CMD_ACTIVATE:
+                    case CMD_DEACTIVATE:
+                    case CMD_STATE:
+                        commands.add(str);
+                        break;
+
+                    case CMD_BASE_LINE:
+                        commands.add(CMD_BASE_LINE);
+
+                        if (it.hasNext()) {
+                            baselineAct = it.next().toLowerCase();
+
+                            if (BASELINE_ADD.equals(baselineAct) || BASELINE_REMOVE.equals(baselineAct) ||
+                                BASELINE_SET.equals(baselineAct) || BASELINE_SET_VERSION.equals(baselineAct))
+                                baselineArgs = nextArg(it, "Expected baseline arguments");
+                            else
+                                throw new IllegalArgumentException("Unexpected argument for " + CMD_BASE_LINE + ": "
+                                        + baselineAct);
+                        }
+
+                }
+            }
+
+            int sz = commands.size();
+
+            if (sz < 1)
+                throw new IllegalArgumentException("No action was specified");
+
+            if (sz > 1)
+                throw new IllegalArgumentException("Only one action can be specified, but found: " + sz);
+
+            GridClientConfiguration cfg = new GridClientConfiguration();
+
+            cfg.setServers(Collections.singletonList(host + ":" + port));
+
+            try (GridClient client = GridClientFactory.start(cfg)) {
+                String cmd = commands.get(0);
+
+                switch (cmd) {
+                    case CMD_ACTIVATE:
+                        activate(client);
+                        break;
+
+                    case CMD_DEACTIVATE:
+                        deactivate(client);
+                        break;
+
+                    case CMD_STATE:
+                        state(client);
+                        break;
+
+                    case CMD_BASE_LINE:
+                        baseline(client, baselineAct, baselineArgs);
+                        break;
+                }
+            }
+
+            return 0;
+        }
+        catch (IllegalArgumentException e) {
+            return error(EXIT_CODE_INVALID_ARGUMENTS, "Check arguments.", e);
+        }
+        catch (Throwable e) {
+            if (isAuthError(e))
+                return error(ERR_AUTHENTICATION_FAILED, "Authentication error.", e);
+
+            if (isConnectionError(e))
+                return error(EXIT_CODE_CONNECTION_FAILED, "Connection to cluster failed.", e);
+
+
+            return error(EXIT_CODE_UNEXPECTED_ERROR, "", e);
+        }
+    }
+
+    /**
+     * @param args Arguments to parse and apply.
+     */
+    public static void main(String[] args) {
+        CommandHandler hnd = new CommandHandler();
+
+        System.exit(hnd.execute(args));
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
index a151eb5..74f5a10 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
@@ -558,6 +558,20 @@ public abstract class GridManagerAdapter<T extends IgniteSpi> implements GridMan
                         return null;
                     }
 
+                    @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode node, DiscoveryDataBag discoData) {
+                        for (GridComponent comp : ctx) {
+                            if (comp.discoveryDataType() == null)
+                                continue;
+
+                            IgniteNodeValidationResult err = comp.validateNode(node, discoData.newJoinerDiscoveryData(comp.discoveryDataType().ordinal()));
+
+                            if (err != null)
+                                return err;
+                        }
+
+                        return null;
+                    }
+
                     @Override public Collection<SecuritySubject> authenticatedSubjects() {
                         try {
                             return ctx.security().authenticatedSubjects();
@@ -705,6 +719,11 @@ public abstract class GridManagerAdapter<T extends IgniteSpi> implements GridMan
     }
 
     /** {@inheritDoc} */
+    @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode node, DiscoveryDataBag.JoiningNodeDiscoveryData discoData) {
+        return null;
+    }
+
+    /** {@inheritDoc} */
     @Override public final String toString() {
         return S.toString(GridManagerAdapter.class, this, "name", getClass().getName());
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/ConsistentIdMapper.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/ConsistentIdMapper.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/ConsistentIdMapper.java
index c524331..59f773d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/ConsistentIdMapper.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/ConsistentIdMapper.java
@@ -19,10 +19,11 @@ package org.apache.ignite.internal.managers.discovery;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Map;
 import java.util.UUID;
-import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.Nullable;
 
@@ -31,71 +32,104 @@ import org.jetbrains.annotations.Nullable;
  */
 public class ConsistentIdMapper {
     /** Discovery manager. */
-    private final GridDiscoveryManager discoveryManager;
+    private final GridDiscoveryManager discoveryMgr;
 
     /**
      * Create an instance of mapper.
      *
-     * @param discoveryManager Discovery manager.
+     * @param discoveryMgr Discovery manager.
      */
-    public ConsistentIdMapper(GridDiscoveryManager discoveryManager) {
-        this.discoveryManager = discoveryManager;
+    public ConsistentIdMapper(GridDiscoveryManager discoveryMgr) {
+        this.discoveryMgr = discoveryMgr;
     }
 
     /**
-     * Map UUID to consistent id.
+     * Maps UUID to compact ID for given baseline topology.
      *
      * @param topVer Topology version.
      * @param nodeId UUID of node.
-     * @return Consistent id of node.
+     * @return Compact ID of node for given baseline topology.
      */
-    public Object mapToConsistentId(AffinityTopologyVersion topVer, UUID nodeId) {
-        ClusterNode node = discoveryManager.node(topVer, nodeId);
+    public short mapToCompactId(AffinityTopologyVersion topVer, UUID nodeId) {
+        Map<UUID, Short> m = discoveryMgr.consistentId(topVer);
 
-        if (node == null)
-            throw new IllegalStateException("Unable to find node by UUID [nodeId=" + nodeId + ", topVer=" + topVer + ']');
+        if (m == null)
+            throw new IllegalStateException("Unable to find consistent id map [topVer" + topVer + ']');
 
-        return node.consistentId();
+        Short constId = m.get(nodeId);
+
+        if (constId == null)
+            throw new IllegalStateException("Unable to find consistentId by UUID [nodeId=" + nodeId + ", topVer=" + topVer + ']');
+
+        return constId;
     }
 
     /**
-     * Map consistent id to UUID.
+     * Maps UUID to compact ID for given baseline topology.
      *
-     * @param consistentId Consistent id of node.
-     * @return UUID of node.
+     * @param topVer Topology version.
+     * @param nodeConstId UUID of node.
+     * @return Compact ID of node for given baseline topology.
      */
-    @Nullable public UUID mapToUUID(Object consistentId) {
-        for (ClusterNode node : discoveryManager.allNodes())
-            if (node.consistentId().equals(consistentId))
-                return node.id();
+    public UUID mapToUuid(AffinityTopologyVersion topVer, short nodeConstId) {
+        Map<Short, UUID> map = discoveryMgr.nodeIdMap(topVer);
+
+        if (map == null)
+            return null;
+
+        UUID constId = map.get(nodeConstId);
 
-        return null;
+        if (constId == null)
+            throw new IllegalStateException("Unable to find UUID by constId [nodeId=" + nodeConstId + ", topVer=" + topVer + ']');
+
+        return constId;
     }
 
     /**
-     * Map primary -> backup node UUIDs to consistent ids.
+     * Map primary -> backup node compact ID accordingly to baseline topology..
      *
      * @param txNodes Primary -> backup UUID nodes.
-     * @return Primary -> backup consistent id nodes.
+     * @return Primary -> backup compact ID nodes.
      */
-    public Map<Object, Collection<Object>> mapToConsistentIds(AffinityTopologyVersion topVer, @Nullable Map<UUID, Collection<UUID>> txNodes) {
+    public Map<Short, Collection<Short>> mapToCompactIds(
+        AffinityTopologyVersion topVer,
+        @Nullable Map<UUID, Collection<UUID>> txNodes,
+        BaselineTopology baselineTop
+    ) {
         if (txNodes == null)
             return null;
 
-        Map<Object, Collection<Object>> consistentMap = U.newHashMap(txNodes.keySet().size());
+        Map<Object, Short> constIdMap = baselineTop.consistentIdMapping();
+
+        Map<UUID, Short> m = discoveryMgr.consistentId(topVer);
 
-        for (UUID node : txNodes.keySet()) {
-            Collection<UUID> backupNodes = txNodes.get(node);
+        int bltNodes = m.size();
 
-            Collection<Object> consistentIdsBackups = new ArrayList<>(backupNodes.size());
+        Map<Short, Collection<Short>> consistentMap = U.newHashMap(txNodes.size());
 
-            for (UUID backup : backupNodes)
-                consistentIdsBackups.add(mapToConsistentId(topVer, backup));
+        int nodeCnt = 0;
 
-            consistentMap.put(mapToConsistentId(topVer, node), consistentIdsBackups);
+        for (Map.Entry<UUID, Collection<UUID>> e : txNodes.entrySet()) {
+            UUID node = e.getKey();
+
+            Collection<UUID> backupNodes = e.getValue();
+
+            Collection<Short> backups = new ArrayList<>(backupNodes.size());
+
+            for (UUID backup : backupNodes) {
+                if (m.containsKey(backup))
+                    nodeCnt++;
+
+                backups.add(mapToCompactId(topVer, backup));
+            }
+
+            // Optimization for short store full nodes set.
+            if (backups.size() == nodeCnt && nodeCnt == (bltNodes - 1))
+                backups = Collections.singletonList(Short.MAX_VALUE);
+
+            consistentMap.put(mapToCompactId(topVer, node), backups);
         }
 
         return consistentMap;
     }
-
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
index 9ed70aa..aa47168 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
@@ -23,11 +23,13 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import org.apache.ignite.cluster.BaselineNode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.P1;
 import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -40,6 +42,13 @@ import org.jetbrains.annotations.Nullable;
  */
 public class DiscoCache {
     /** */
+    private static final C1<BaselineNode, ClusterNode> BASELINE_TO_CLUSTER = new C1<BaselineNode, ClusterNode>() {
+        @Override public ClusterNode apply(BaselineNode baselineNode) {
+            return (ClusterNode)baselineNode;
+        }
+    };
+
+    /** */
     private final DiscoveryDataClusterState state;
 
     /** Local node. */
@@ -57,6 +66,9 @@ public class DiscoCache {
     /** Daemon nodes. */
     private final List<ClusterNode> daemonNodes;
 
+    /** Baseline nodes. */
+    private final List<? extends BaselineNode> baselineNodes;
+
     /** All remote nodes with at least one cache configured. */
     @GridToStringInclude
     private final List<ClusterNode> rmtNodesWithCaches;
@@ -81,6 +93,18 @@ public class DiscoCache {
     /** */
     private final AffinityTopologyVersion topVer;
 
+    /** */
+    final Map<UUID, Short> nodeIdToConsIdx;
+
+    /** */
+    final Map<Short, UUID> consIdxToNodeId;
+
+    /** */
+    private final P1<BaselineNode> aliveBaselineNodePred;
+
+    /** */
+    private final P1<ClusterNode> aliveNodePred;
+
     /**
      * @param topVer Topology version.
      * @param state Current cluster state.
@@ -90,10 +114,13 @@ public class DiscoCache {
      * @param srvNodes Server nodes.
      * @param daemonNodes Daemon nodes.
      * @param rmtNodesWithCaches Remote nodes with at least one cache configured.
+     * @param baselineNodes Baseline nodes or {@code null} if baseline was not set.
      * @param allCacheNodes Cache nodes by cache name.
      * @param cacheGrpAffNodes Affinity nodes by cache group ID.
      * @param nodeMap Node map.
-     * @param alives Alive nodes.
+     * @param alives0 Alive nodes.
+     * @param nodeIdToConsIdx Node ID to consistent ID mapping.
+     * @param consIdxToNodeId Consistent ID to node ID mapping.
      * @param minNodeVer Minimum node version.
      */
     DiscoCache(
@@ -105,11 +132,15 @@ public class DiscoCache {
         List<ClusterNode> srvNodes,
         List<ClusterNode> daemonNodes,
         List<ClusterNode> rmtNodesWithCaches,
+        @Nullable List<? extends BaselineNode> baselineNodes,
         Map<Integer, List<ClusterNode>> allCacheNodes,
         Map<Integer, List<ClusterNode>> cacheGrpAffNodes,
         Map<UUID, ClusterNode> nodeMap,
-        Set<UUID> alives,
-        IgniteProductVersion minNodeVer) {
+        Set<UUID> alives0,
+        @Nullable Map<UUID, Short> nodeIdToConsIdx,
+        @Nullable  Map<Short, UUID> consIdxToNodeId,
+        IgniteProductVersion minNodeVer
+    ) {
         this.topVer = topVer;
         this.state = state;
         this.loc = loc;
@@ -118,11 +149,28 @@ public class DiscoCache {
         this.srvNodes = srvNodes;
         this.daemonNodes = daemonNodes;
         this.rmtNodesWithCaches = rmtNodesWithCaches;
+        this.baselineNodes = baselineNodes;
         this.allCacheNodes = allCacheNodes;
         this.cacheGrpAffNodes = cacheGrpAffNodes;
         this.nodeMap = nodeMap;
-        this.alives.addAll(alives);
+        alives.addAll(alives0);
         this.minNodeVer = minNodeVer;
+        this.nodeIdToConsIdx = nodeIdToConsIdx;
+        this.consIdxToNodeId = consIdxToNodeId;
+
+        aliveBaselineNodePred = new P1<BaselineNode>() {
+            @Override
+            public boolean apply(BaselineNode node) {
+                return node instanceof ClusterNode && alives.contains(((ClusterNode)node).id());
+
+            }
+        };
+
+        aliveNodePred = new P1<ClusterNode>() {
+            @Override public boolean apply(ClusterNode node) {
+                return alives.contains(node.id());
+            }
+        };
     }
 
     /**
@@ -156,6 +204,15 @@ public class DiscoCache {
         return rmtNodes;
     }
 
+    /**
+     * Returns a collection of baseline nodes.
+     *
+     * @return A collection of baseline nodes or {@code null} if baseline topology was not set.
+     */
+    @Nullable public List<? extends BaselineNode> baselineNodes() {
+        return baselineNodes;
+    }
+
     /** @return All nodes. */
     public List<ClusterNode> allNodes() {
         return allNodes;
@@ -171,17 +228,23 @@ public class DiscoCache {
         return daemonNodes;
     }
 
+    /** @return Consistent id map UUID -> Short (compacted consistent id). */
+    public Map<UUID, Short> consistentIdMap() {
+        return nodeIdToConsIdx;
+    }
+
+    /** @return Consistent id map Short (compacted consistent id) -> UUID. */
+    public Map<Short, UUID> nodeIdMap() {
+        return consIdxToNodeId;
+    }
+
     /**
      * Gets all alive remote nodes that have at least one cache configured.
      *
      * @return Collection of nodes.
      */
     public Collection<ClusterNode> remoteAliveNodesWithCaches() {
-        return F.view(rmtNodesWithCaches, new P1<ClusterNode>() {
-            @Override public boolean apply(ClusterNode node) {
-                return alives.contains(node.id());
-            }
-        });
+        return F.view(rmtNodesWithCaches, aliveNodePred);
     }
 
     /**
@@ -190,17 +253,26 @@ public class DiscoCache {
      * @return Collection of nodes.
      */
     public Collection<ClusterNode> aliveServerNodes() {
-        return F.view(serverNodes(), new P1<ClusterNode>() {
-            @Override public boolean apply(ClusterNode node) {
-                return alives.contains(node.id());
-            }
-        });
+        return F.view(serverNodes(), aliveNodePred);
+    }
+
+    /**
+     * Returns a collection of live baseline nodes.
+     *
+     * @return A view of baseline nodes that are currently present in the cluster or {@code null} if baseline
+     *      topology was not set.
+     */
+    @Nullable public Collection<ClusterNode> aliveBaselineNodes() {
+        return baselineNodes == null ? null : F.viewReadOnly(baselineNodes, BASELINE_TO_CLUSTER, aliveBaselineNodePred);
+
     }
 
     /**
      * @return Oldest alive server node.
      */
-    public @Nullable ClusterNode oldestAliveServerNode(){
+    @SuppressWarnings("ForLoopReplaceableByForEach")
+    @Nullable public ClusterNode oldestAliveServerNode(){
+        // Avoid iterator allocation.
         for (int i = 0; i < srvNodes.size(); i++) {
             ClusterNode srv = srvNodes.get(i);
 
@@ -330,10 +402,13 @@ public class DiscoCache {
             srvNodes,
             daemonNodes,
             rmtNodesWithCaches,
+            baselineNodes,
             allCacheNodes,
             cacheGrpAffNodes,
             nodeMap,
             alives,
+            nodeIdToConsIdx,
+            consIdxToNodeId,
             minNodeVer);
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 088c9ce..65cc666 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -50,6 +50,7 @@ import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteInterruptedException;
 import org.apache.ignite.cache.CacheMetrics;
 import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.BaselineNode;
 import org.apache.ignite.cluster.ClusterMetrics;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -59,10 +60,10 @@ import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.GridComponent;
 import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.GridNodeOrderComparator;
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.cluster.NodeOrderComparator;
 import org.apache.ignite.internal.events.DiscoveryCustomEvent;
 import org.apache.ignite.internal.managers.GridManagerAdapter;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
@@ -74,12 +75,12 @@ import org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
 import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage;
 import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage;
 import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
-import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor;
+import org.apache.ignite.internal.processors.cluster.IGridClusterStateProcessor;
 import org.apache.ignite.internal.processors.security.SecurityContext;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
 import org.apache.ignite.internal.util.GridAtomicLong;
@@ -705,11 +706,19 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
                     Snapshot snapshot = topSnap.get();
 
                     if (customMsg == null) {
-                        discoCache = createDiscoCache(nextTopVer,
+                        discoCache = createDiscoCache(
+                            nextTopVer,
                             ctx.state().clusterState(),
                             locNode,
                             topSnapshot);
                     }
+                    else if (customMsg instanceof ChangeGlobalStateMessage) {
+                        discoCache = createDiscoCache(
+                            nextTopVer,
+                            ctx.state().pendingState((ChangeGlobalStateMessage)customMsg),
+                            locNode,
+                            topSnapshot);
+                    }
                     else
                         discoCache = customMsg.createDiscoCache(GridDiscoveryManager.this, nextTopVer, snapshot.discoCache);
 
@@ -784,7 +793,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
 
                     topSnap.set(new Snapshot(AffinityTopologyVersion.ZERO,
                         createDiscoCache(AffinityTopologyVersion.ZERO, ctx.state().clusterState(), locNode,
-                            Collections.<ClusterNode>singleton(locNode))
+                            Collections.singleton(locNode))
                     ));
                 }
                 else if (type == EVT_CLIENT_NODE_RECONNECTED) {
@@ -846,7 +855,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
 
                 if (ctx.localNodeId().equals(dataBag.joiningNodeId())) {
                     // NodeAdded msg reached joining node after round-trip over the ring.
-                    GridClusterStateProcessor stateProc = ctx.state();
+                    IGridClusterStateProcessor stateProc = ctx.state();
 
                     stateProc.onGridDataReceived(dataBag.gridDiscoveryData(
                         stateProc.discoveryDataType().ordinal()));
@@ -858,7 +867,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
                 }
                 else {
                     // Discovery data from newly joined node has to be applied to the current old node.
-                    GridClusterStateProcessor stateProc = ctx.state();
+                    IGridClusterStateProcessor stateProc = ctx.state();
 
                     JoiningNodeDiscoveryData data0 = dataBag.newJoinerDiscoveryData(
                         stateProc.discoveryDataType().ordinal());
@@ -1810,6 +1819,15 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
     }
 
     /**
+     * @param topVer Topology version.
+     * @return All baseline nodes for given topology version or {@code null} if baseline was not set for the
+     *      given topology version.
+     */
+    @Nullable public List<? extends BaselineNode> baselineNodes(AffinityTopologyVersion topVer) {
+        return resolveDiscoCache(CU.cacheId(null), topVer).baselineNodes();
+    }
+
+    /**
      * Gets node from history for given topology version.
      *
      * @param topVer Topology version.
@@ -1821,6 +1839,26 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
     }
 
     /**
+     * Gets consistentId from history for given topology version.
+     *
+     * @param topVer Topology version.
+     * @return Compacted consistent id.
+     */
+    public Map<UUID, Short> consistentId(AffinityTopologyVersion topVer) {
+        return resolveDiscoCache(CU.cacheId(null), topVer).consistentIdMap();
+    }
+
+    /**
+     * Gets consistentId from history for given topology version.
+     *
+     * @param topVer Topology version.
+     * @return Compacted consistent id map.
+     */
+    public  Map<Short, UUID> nodeIdMap(AffinityTopologyVersion topVer) {
+        return resolveDiscoCache(CU.cacheId(null), topVer).nodeIdMap();
+    }
+
+    /**
      * Gets cache nodes for cache with given name.
      *
      * @param cacheName Cache name.
@@ -2004,7 +2042,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
 
     /**
      * @return Consistent ID.
-     * @deprecated Use PdsConsistentIdProcessor to get actual consistent ID
+     * @deprecated Use {@link ClusterNode#consistentId()} of local node to get actual consistent ID.
      */
     @Deprecated
     public Serializable consistentId() {
@@ -2206,6 +2244,10 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
         ArrayList<ClusterNode> rmtNodes = new ArrayList<>(topSnapshot.size());
         ArrayList<ClusterNode> allNodes = new ArrayList<>(topSnapshot.size());
 
+        Map<UUID, Short> nodeIdToConsIdx;
+        Map<Short, UUID> consIdxToNodeId;
+        List<? extends BaselineNode> baselineNodes;
+
         IgniteProductVersion minVer = null;
 
         for (ClusterNode node : topSnapshot) {
@@ -2237,10 +2279,52 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
 
         Map<Integer, List<ClusterNode>> allCacheNodes = U.newHashMap(allNodes.size());
         Map<Integer, List<ClusterNode>> cacheGrpAffNodes = U.newHashMap(allNodes.size());
-        Set<ClusterNode> rmtNodesWithCaches = new TreeSet<>(GridNodeOrderComparator.INSTANCE);
+        Set<ClusterNode> rmtNodesWithCaches = new TreeSet<>(NodeOrderComparator.getInstance());
 
         fillAffinityNodeCaches(allNodes, allCacheNodes, cacheGrpAffNodes, rmtNodesWithCaches);
 
+        BaselineTopology blt = state.baselineTopology();
+
+        if (blt != null) {
+            nodeIdToConsIdx = U.newHashMap(srvNodes.size());
+            consIdxToNodeId = U.newHashMap(srvNodes.size());
+
+            Map<Object, Short> m = blt.consistentIdMapping();
+
+            Map<Object, ClusterNode> aliveNodesByConsId = U.newHashMap(srvNodes.size());
+
+            for (ClusterNode node : srvNodes) {
+                Short compactedId = m.get(node.consistentId());
+
+                if (compactedId != null) {
+                    nodeIdToConsIdx.put(node.id(), compactedId);
+
+                    consIdxToNodeId.put(compactedId, node.id());
+                }
+
+                aliveNodesByConsId.put(node.consistentId(), node);
+            }
+
+            List<BaselineNode >baselineNodes0 = new ArrayList<>(blt.size());
+
+            for (Object consId : blt.consistentIds()) {
+                ClusterNode srvNode = aliveNodesByConsId.get(consId);
+
+                if (srvNode != null)
+                    baselineNodes0.add(srvNode);
+                else
+                    baselineNodes0.add(blt.baselineNode(consId));
+            }
+
+            baselineNodes = baselineNodes0;
+        }
+        else {
+            nodeIdToConsIdx = null;
+            consIdxToNodeId = null;
+
+            baselineNodes = null;
+        }
+
         return new DiscoCache(
             topVer,
             state,
@@ -2250,10 +2334,13 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
             Collections.unmodifiableList(srvNodes),
             Collections.unmodifiableList(daemonNodes),
             U.sealList(rmtNodesWithCaches),
+            baselineNodes == null ? null : Collections.unmodifiableList(baselineNodes),
             Collections.unmodifiableMap(allCacheNodes),
             Collections.unmodifiableMap(cacheGrpAffNodes),
             Collections.unmodifiableMap(nodeMap),
             alives,
+            nodeIdToConsIdx == null ? null : Collections.unmodifiableMap(nodeIdToConsIdx),
+            consIdxToNodeId == null ? null : Collections.unmodifiableMap(consIdxToNodeId),
             minVer);
     }
 
@@ -2364,8 +2451,12 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
                         discoWrk.addEvent(EVT_NODE_SEGMENTED,
                             AffinityTopologyVersion.NONE,
                             node,
-                            createDiscoCache(AffinityTopologyVersion.NONE, null, node, locNodeOnlyTop),
-                            locNodeOnlyTop,
+                            createDiscoCache(
+                                AffinityTopologyVersion.NONE,
+                                ctx.state().clusterState(),
+                                node,
+                                locNodeOnlyTop
+                            ), locNodeOnlyTop,
                             null);
 
                         lastSegChkRes.set(false);
@@ -3056,12 +3147,14 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
      * @param discoCache Current disco cache.
      * @return New discovery cache.
      */
-    public DiscoCache createDiscoCacheOnCacheChange(AffinityTopologyVersion topVer,
-        DiscoCache discoCache) {
+    public DiscoCache createDiscoCacheOnCacheChange(
+        AffinityTopologyVersion topVer,
+        DiscoCache discoCache
+    ) {
         List<ClusterNode> allNodes = discoCache.allNodes();
         Map<Integer, List<ClusterNode>> allCacheNodes = U.newHashMap(allNodes.size());
         Map<Integer, List<ClusterNode>> cacheGrpAffNodes = U.newHashMap(allNodes.size());
-        Set<ClusterNode> rmtNodesWithCaches = new TreeSet<>(GridNodeOrderComparator.INSTANCE);
+        Set<ClusterNode> rmtNodesWithCaches = new TreeSet<>(NodeOrderComparator.getInstance());
 
         fillAffinityNodeCaches(allNodes, allCacheNodes, cacheGrpAffNodes, rmtNodesWithCaches);
 
@@ -3074,10 +3167,13 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
             discoCache.serverNodes(),
             discoCache.daemonNodes(),
             U.sealList(rmtNodesWithCaches),
+            discoCache.baselineNodes(),
             allCacheNodes,
             cacheGrpAffNodes,
             discoCache.nodeMap,
             discoCache.alives,
+            discoCache.nodeIdToConsIdx,
+            discoCache.consIdxToNodeId,
             discoCache.minimumNodeVersion());
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java
index bf6e807..276e10e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java
@@ -57,13 +57,15 @@ public class UnsafeMemoryProvider implements DirectMemoryProvider {
 
     /** {@inheritDoc} */
     @Override public void shutdown() {
-        for (Iterator<DirectMemoryRegion> it = regions.iterator(); it.hasNext(); ) {
-            DirectMemoryRegion chunk = it.next();
+        if (regions != null) {
+            for (Iterator<DirectMemoryRegion> it = regions.iterator(); it.hasNext(); ) {
+                DirectMemoryRegion chunk = it.next();
 
-            GridUnsafe.freeMemory(chunk.address());
+                GridUnsafe.freeMemory(chunk.address());
 
-            // Safety.
-            it.remove();
+                // Safety.
+                it.remove();
+            }
         }
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
index 64c5927..6802a3f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
@@ -52,6 +52,11 @@ public interface IgnitePageStoreManager extends GridCacheSharedManager, IgniteCh
         throws IgniteCheckedException;
 
     /**
+     * Initializes disk cache store structures.
+     */
+    public void initializeForMetastorage() throws IgniteCheckedException;
+
+    /**
      * Callback called when a cache is stopping. After this callback is invoked, no data associated with
      * the given cache will be stored on disk.
      *

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
index 42d9611..19b47e6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
@@ -96,10 +96,11 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni
      * the given pointer will be kept because there is a configurable WAL history size. Those entries may be used
      * for partial partition rebalancing.
      *
-     * @param ptr Pointer for which it is safe to clear the log.
+     * @param low Pointer since which WAL will be truncated. If null, WAL will be truncated from the oldest segment.
+     * @param high Pointer for which it is safe to clear the log.
      * @return Number of deleted WAL segments.
      */
-    public int truncate(WALPointer ptr);
+    public int truncate(WALPointer low, WALPointer high);
 
     /**
      * Gives a hint to WAL manager to compact WAL until given pointer (exclusively).

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/BaselineTopologyRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/BaselineTopologyRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/BaselineTopologyRecord.java
new file mode 100644
index 0000000..48b60b3
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/BaselineTopologyRecord.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.pagemem.wal.record;
+
+import java.util.Map;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Record for storing baseline topology compact node ID to consistent node ID mapping.
+ */
+public class BaselineTopologyRecord extends WALRecord {
+    /** Id. */
+    private int id;
+
+    /** Compact ID to consistent ID mapping. */
+    private Map<Short, Object> mapping;
+
+    /**
+     * Default constructor.
+     */
+    private BaselineTopologyRecord() {
+        // No-op, used from factory methods.
+    }
+
+    /**
+     * @param id Baseline topology ID.
+     * @param mapping Compact ID to consistent ID mapping.
+     */
+    public BaselineTopologyRecord(int id, Map<Short, Object> mapping) {
+        this.id = id;
+        this.mapping = mapping;
+    }
+
+    /** {@inheritDoc} */
+    @Override public RecordType type() {
+        return RecordType.BASELINE_TOP_RECORD;
+    }
+
+    /**
+     * Returns baseline topology ID.
+     *
+     * @return Baseline topology ID.
+     */
+    public int id() {
+        return id;
+    }
+
+    /**
+     * Returns mapping.
+     *
+     * @return Compact ID to consistent ID mapping.
+     */
+    public Map<Short, Object> mapping() {
+        return mapping;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(BaselineTopologyRecord.class, this, "super", super.toString());
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/CacheState.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/CacheState.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/CacheState.java
index 41d38d0..49025c7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/CacheState.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/CacheState.java
@@ -30,6 +30,9 @@ public class CacheState {
     private long[] vals;
 
     /** */
+    private byte[] states;
+
+    /** */
     private int idx;
 
     /**
@@ -38,6 +41,7 @@ public class CacheState {
     public CacheState(int partsCnt) {
         parts = new int[partsCnt];
         vals = new long[partsCnt * 2];
+        states = new byte[partsCnt];
     }
 
     /**
@@ -46,6 +50,16 @@ public class CacheState {
      * @param cntr Partition counter.
      */
     public void addPartitionState(int partId, long size, long cntr) {
+        addPartitionState(partId, size, cntr, (byte)-1);
+    }
+
+    /**
+     * @param partId Partition ID to add.
+     * @param size Partition size.
+     * @param cntr Partition counter.
+     * @param state Partition state.
+     */
+    public void addPartitionState(int partId, long size, long cntr, byte state) {
         if (idx == parts.length)
             throw new IllegalStateException("Failed to add new partition to the partitions state " +
                 "(no enough space reserved) [partId=" + partId + ", reserved=" + parts.length + ']');
@@ -57,6 +71,8 @@ public class CacheState {
         }
 
         parts[idx] = partId;
+        states[idx] = state;
+
         vals[2 * idx] = size;
         vals[2 * idx + 1] = cntr;
 
@@ -97,6 +113,14 @@ public class CacheState {
 
     /**
      * @param idx Index to get.
+     * @return State partition.
+     */
+    public byte stateByIndex(int idx) {
+        return states[idx];
+    }
+
+    /**
+     * @param idx Index to get.
      * @return Partition size by index.
      */
     public long partitionSizeByIndex(int idx) {

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataEntry.java
index cb6b482..3511aff 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataEntry.java
@@ -97,8 +97,8 @@ public class DataEntry {
         this.partId = partId;
         this.partCnt = partCnt;
 
-        // Only CREATE, UPDATE and DELETE operations should be stored in WAL.
-        assert op == GridCacheOperation.CREATE || op == GridCacheOperation.UPDATE || op == GridCacheOperation.DELETE : op;
+        // Only READ, CREATE, UPDATE and DELETE operations should be stored in WAL.
+        assert op == GridCacheOperation.READ || op == GridCacheOperation.CREATE || op == GridCacheOperation.UPDATE || op == GridCacheOperation.DELETE : op;
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java
index ac569bd..7a4d6b8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java
@@ -21,6 +21,7 @@ import java.util.Collections;
 import java.util.List;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
  * Logical data record with cache operation description.
@@ -48,19 +49,18 @@ public class DataRecord extends TimeStampRecord {
      * @param writeEntry Write entry.
      */
     public DataRecord(DataEntry writeEntry) {
-        this(Collections.singletonList(writeEntry));
+        this(writeEntry, U.currentTimeMillis());
     }
 
     /**
      * @param writeEntries Write entries.
      */
     public DataRecord(List<DataEntry> writeEntries) {
-        this.writeEntries = writeEntries;
+        this(writeEntries, U.currentTimeMillis());
     }
 
     /**
      * @param writeEntry Write entry.
-     * @param timestamp TimeStamp.
      */
     public DataRecord(DataEntry writeEntry, long timestamp) {
         this(Collections.singletonList(writeEntry), timestamp);

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/ExchangeRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/ExchangeRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/ExchangeRecord.java
new file mode 100644
index 0000000..87bc4e6
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/ExchangeRecord.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.pagemem.wal.record;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Partition map exchange WAL record.
+ */
+public class ExchangeRecord extends TimeStampRecord {
+    /** Event. */
+    private Short constId;
+
+    /** Type. */
+    private Type type;
+
+    /**
+     * @param constId Const id.
+     * @param type Type.
+     * @param timeStamp TimeStamp.
+     */
+    public ExchangeRecord(Short constId, Type type, long timeStamp) {
+        super(timeStamp);
+
+        this.constId = constId;
+        this.type = type;
+    }
+
+    /**
+     * @param constId Const id.
+     * @param type Type.
+     */
+    public ExchangeRecord(Short constId, Type type) {
+        this.constId = constId;
+        this.type = type;
+    }
+
+    /** {@inheritDoc} */
+    @Override public RecordType type() {
+        return RecordType.EXCHANGE;
+    }
+
+    /**
+     *
+     */
+    public Short getConstId() {
+        return constId;
+    }
+
+    /**
+     *
+     */
+    public Type getType() {
+        return type;
+    }
+
+    /**
+     *
+     */
+    public enum Type {
+        /** Join. */
+        JOIN,
+        /** Left. */
+        LEFT
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ExchangeRecord.class, this, "super", super.toString());
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/FilteredRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/FilteredRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/FilteredRecord.java
index 519e825..e077e5a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/FilteredRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/FilteredRecord.java
@@ -24,6 +24,9 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRe
  * {@link AbstractWalRecordsIterator}.
  */
 public class FilteredRecord extends WALRecord {
+    /** Instance. */
+    public static final FilteredRecord INSTANCE = new FilteredRecord();
+
     /** {@inheritDoc} */
     @Override public RecordType type() {
         return null;

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java
new file mode 100644
index 0000000..e269de2
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.pagemem.wal.record;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public class MetastoreDataRecord extends WALRecord {
+    /** */
+    private final String key;
+
+    /** */
+    @Nullable private final byte[] value;
+
+    /**
+     * @param key Key.
+     * @param value Value.
+     */
+    public MetastoreDataRecord(String key, @Nullable byte[] value) {
+        this.key = key;
+        this.value = value;
+    }
+
+    /** */
+    public String key() {
+        return key;
+    }
+
+    /** */
+    @Nullable public byte[] value() {
+        return value;
+    }
+
+    /** {@inheritDoc} */
+    @Override public RecordType type() {
+        return RecordType.METASTORE_DATA_RECORD;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(MetastoreDataRecord.class, this, "super", super.toString());
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/PageSnapshot.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/PageSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/PageSnapshot.java
index 6ee96a4..6740221 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/PageSnapshot.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/PageSnapshot.java
@@ -25,6 +25,7 @@ import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import sun.nio.ch.DirectBuffer;
 
 /**
  *
@@ -92,9 +93,12 @@ public class PageSnapshot extends WALRecord {
                 + "],\nsuper = ["
                 + super.toString() + "]]";
         }
-        catch (IgniteCheckedException e) {
+        catch (IgniteCheckedException ignored) {
             return "Error during call'toString' of PageSnapshot [fullPageId=" + fullPageId() +
                 ", pageData = " + Arrays.toString(pageData) + ", super=" + super.toString() + "]";
         }
+        finally {
+            ((DirectBuffer)buf).cleaner().clean();
+        }
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/SnapshotRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/SnapshotRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/SnapshotRecord.java
index 3c3a77b..c6b6329 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/SnapshotRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/SnapshotRecord.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.pagemem.wal.record;
 
+import org.apache.ignite.internal.util.typedef.internal.S;
+
 /**
  * Wal snapshot record.
  */
@@ -49,10 +51,20 @@ public class SnapshotRecord extends WALRecord {
         return full;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean rollOver() {
+        return true;
+    }
+
     /**
      *
      */
     @Override public RecordType type() {
         return RecordType.SNAPSHOT;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(SnapshotRecord.class, this, "super", super.toString());
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TimeStampRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TimeStampRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TimeStampRecord.java
index 3f29dfd..c1b8584 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TimeStampRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TimeStampRecord.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.pagemem.wal.record;
 
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
@@ -25,6 +27,7 @@ import org.apache.ignite.internal.util.typedef.internal.U;
  */
 public abstract class TimeStampRecord extends WALRecord {
     /** Timestamp. */
+    @GridToStringInclude
     protected long timestamp;
 
     /**
@@ -54,4 +57,9 @@ public abstract class TimeStampRecord extends WALRecord {
     public long timestamp() {
         return timestamp;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(TimeStampRecord.class, this);
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TxRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TxRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TxRecord.java
index f933fa9..90df004 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TxRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TxRecord.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.pagemem.wal.record;
 import java.util.Collection;
 import java.util.Map;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.transactions.TransactionState;
 import org.jetbrains.annotations.Nullable;
@@ -30,9 +31,11 @@ import org.jetbrains.annotations.Nullable;
  */
 public class TxRecord extends TimeStampRecord {
     /** Transaction state. */
+    @GridToStringInclude
     private TransactionState state;
 
     /** Global transaction identifier within cluster, assigned by transaction coordinator. */
+    @GridToStringInclude
     private GridCacheVersion nearXidVer;
 
     /** Transaction entries write topology version. */
@@ -42,33 +45,27 @@ public class TxRecord extends TimeStampRecord {
      * Transaction participating nodes.
      *
      * Structure:
-     * Primary node -> [Backup nodes...]
+     * Primary node -> [Backup nodes...], where nodes are identified by compact ID for some baseline topology.
      **/
-    @Nullable private Map<Object, Collection<Object>> participatingNodes;
-
-    /** If transaction is remote, primary node for this backup node. */
-    @Nullable private Object primaryNode;
+    @Nullable private Map<Short, Collection<Short>> participatingNodes;
 
     /**
      *
      * @param state Transaction state.
      * @param nearXidVer Transaction id.
      * @param writeVer Transaction entries write topology version.
-     * @param participatingNodes Primary -> Backup nodes participating in transaction.
-     * @param primaryNode Primary node.
+     * @param participatingNodes Primary -> Backup nodes compact IDs participating in transaction.
      */
     public TxRecord(
         TransactionState state,
         GridCacheVersion nearXidVer,
         GridCacheVersion writeVer,
-        @Nullable Map<Object, Collection<Object>> participatingNodes,
-        @Nullable Object primaryNode
+        @Nullable Map<Short, Collection<Short>> participatingNodes
     ) {
         this.state = state;
         this.nearXidVer = nearXidVer;
         this.writeVer = writeVer;
         this.participatingNodes = participatingNodes;
-        this.primaryNode = primaryNode;
     }
 
     /**
@@ -76,24 +73,21 @@ public class TxRecord extends TimeStampRecord {
      * @param nearXidVer Transaction id.
      * @param writeVer Transaction entries write topology version.
      * @param participatingNodes Primary -> Backup nodes participating in transaction.
-     * @param primaryNode Primary node.
-     * @param timestamp TimeStamp.
+     * @param ts TimeStamp.
      */
     public TxRecord(
         TransactionState state,
         GridCacheVersion nearXidVer,
         GridCacheVersion writeVer,
-        @Nullable Map<Object, Collection<Object>> participatingNodes,
-        @Nullable Object primaryNode,
-        long timestamp
+        @Nullable Map<Short, Collection<Short>> participatingNodes,
+        long ts
     ) {
-        super(timestamp);
+        super(ts);
 
         this.state = state;
         this.nearXidVer = nearXidVer;
         this.writeVer = writeVer;
         this.participatingNodes = participatingNodes;
-        this.primaryNode = primaryNode;
     }
 
     /** {@inheritDoc} */
@@ -144,33 +138,19 @@ public class TxRecord extends TimeStampRecord {
     }
 
     /**
-     * @return Primary -> backup participating nodes.
+     * @return Primary -> backup participating nodes compact IDs.
      */
-    public Map<Object, Collection<Object>> participatingNodes() {
+    public Map<Short, Collection<Short>> participatingNodes() {
         return participatingNodes;
     }
 
     /**
-     * @param participatingNodeIds Primary -> backup participating nodes.
+     * @param participatingNodeIds Primary -> backup participating nodes compact IDs.
      */
-    public void participatingNodes(Map<Object, Collection<Object>> participatingNodeIds) {
+    public void participatingNodes(Map<Short, Collection<Short>> participatingNodeIds) {
         this.participatingNodes = participatingNodeIds;
     }
 
-    /**
-     * @return Is transaction remote for backup.
-     */
-    public boolean remote() {
-        return primaryNode != null;
-    }
-
-    /**
-     * @return Primary node for backup if transaction is remote.
-     */
-    @Nullable public Object primaryNode() {
-        return primaryNode;
-    }
-
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(TxRecord.class, this, "super", super.toString());

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java
index 7446916..8362a69 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java
@@ -166,7 +166,16 @@ public abstract class WALRecord {
         PARTITION_DESTROY,
 
         /** Snapshot record. */
-        SNAPSHOT;
+        SNAPSHOT,
+
+        /** Metastore data record. */
+        METASTORE_DATA_RECORD,
+
+        /** Exchange record. */
+        EXCHANGE,
+
+        /** Baseline topology record. */
+        BASELINE_TOP_RECORD;
 
         /** */
         private static final RecordType[] VALS = RecordType.values();
@@ -241,6 +250,13 @@ public abstract class WALRecord {
     }
 
     /**
+     * @return Need wal rollOver.
+     */
+    public boolean rollOver(){
+        return false;
+    }
+
+    /**
      * @return Entry type.
      */
     public abstract RecordType type();

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
index e07c388..5324d56 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
@@ -19,7 +19,8 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
 
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
@@ -54,7 +55,7 @@ public class DataPageInsertFragmentRecord extends PageDeltaRecord {
 
     /** {@inheritDoc} */
     @Override public void applyDelta(PageMemory pageMem, long pageAddr) throws IgniteCheckedException {
-        DataPageIO io = DataPageIO.VERSIONS.forPage(pageAddr);
+        AbstractDataPageIO io = PageIO.getPageIO(pageAddr);
 
         io.addRowFragment(pageAddr, payload, lastLink, pageMem.pageSize());
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java
index f315058..2c9a8e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java
@@ -19,7 +19,8 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
 
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
@@ -55,7 +56,7 @@ public class DataPageInsertRecord extends PageDeltaRecord {
     @Override public void applyDelta(PageMemory pageMem, long pageAddr) throws IgniteCheckedException {
         assert payload != null;
 
-        DataPageIO io = DataPageIO.VERSIONS.forPage(pageAddr);
+        AbstractDataPageIO io = PageIO.getPageIO(pageAddr);
 
         io.addRow(pageAddr, payload, pageMem.pageSize());
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java
index 484ec87..f7776be 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java
@@ -19,7 +19,8 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
 
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
@@ -50,7 +51,7 @@ public class DataPageRemoveRecord extends PageDeltaRecord {
     /** {@inheritDoc} */
     @Override public void applyDelta(PageMemory pageMem, long pageAddr)
         throws IgniteCheckedException {
-        DataPageIO io = DataPageIO.VERSIONS.forPage(pageAddr);
+        AbstractDataPageIO io = PageIO.getPageIO(pageAddr);
 
         io.removeRow(pageAddr, itemId, pageMem.pageSize());
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java
index 0ade484..e679611 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java
@@ -19,7 +19,8 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
 
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.DATA_PAGE_SET_FREE_LIST_PAGE;
@@ -51,7 +52,7 @@ public class DataPageSetFreeListPageRecord extends PageDeltaRecord {
 
     /** {@inheritDoc} */
     @Override public void applyDelta(PageMemory pageMem, long pageAddr) throws IgniteCheckedException {
-        DataPageIO io = DataPageIO.VERSIONS.forPage(pageAddr);
+        AbstractDataPageIO io = PageIO.getPageIO(pageAddr);
 
         io.setFreeListPageId(pageAddr, freeListPage);
     }


[05/17] ignite git commit: IEP-4 Baseline topology for persistent caches (Phase 1) Contributed by: Dmitriy Govorukhin Dmitry Pavlov Eduard Shangareev Ily

Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskResult.java
new file mode 100644
index 0000000..a3559bd
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskResult.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.baseline;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Collection;
+import java.util.Map;
+import java.util.TreeMap;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Result for {@link VisorBaselineTask}.
+ */
+public class VisorBaselineTaskResult extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Cluster state. */
+    private boolean active;
+
+    /** Current topology version. */
+    private long topVer;
+
+    /** Current baseline nodes. */
+    private Map<String, VisorBaselineNode> baseline;
+
+    /** Current server nodes. */
+    private Map<String, VisorBaselineNode> servers;
+
+    /**
+     * Default constructor.
+     */
+    public VisorBaselineTaskResult() {
+        // No-op.
+    }
+
+    /**
+     * @param nodes Nodes to process.
+     * @return Map of DTO objects.
+     */
+    private static Map<String, VisorBaselineNode> toMap(Collection<? extends BaselineNode> nodes) {
+        if (F.isEmpty(nodes))
+            return null;
+
+        Map<String, VisorBaselineNode> map = new TreeMap<>();
+
+        for (BaselineNode node : nodes) {
+            VisorBaselineNode dto = new VisorBaselineNode(node);
+
+            map.put(dto.getConsistentId(), dto);
+        }
+
+        return map;
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param active Cluster state.
+     * @param topVer Current topology version.
+     * @param baseline Current baseline nodes.
+     * @param servers Current server nodes.
+     */
+    public VisorBaselineTaskResult(
+        boolean active,
+        long topVer,
+        Collection<? extends BaselineNode> baseline,
+        Collection<? extends BaselineNode> servers) {
+        this.active = active;
+        this.topVer = topVer;
+        this.baseline = toMap(baseline);
+        this.servers = toMap(servers);
+    }
+
+    /**
+     * @return Cluster state.
+     */
+    public boolean isActive() {
+        return active;
+    }
+
+    /**
+     * @return Current topology version.
+     */
+    public long getTopologyVersion() {
+        return topVer;
+    }
+
+    /**
+     * @return Baseline nodes.
+     */
+    public Map<String, VisorBaselineNode> getBaseline() {
+        return baseline;
+    }
+
+    /**
+     * @return Server nodes.
+     */
+    public Map<String, VisorBaselineNode> getServers() {
+        return servers;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        out.writeBoolean(active);
+        out.writeLong(topVer);
+        U.writeMap(out, baseline);
+        U.writeMap(out, servers);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        active = in.readBoolean();
+        topVer = in.readLong();
+        baseline = U.readTreeMap(in);
+        servers = U.readTreeMap(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorBaselineTaskResult.class, this);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
index 48c3abb..4e41537 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
@@ -301,6 +301,9 @@ public class VisorGatewayTask implements ComputeTask<Object[], Object> {
                 return res;
             }
 
+            if (cls.isEnum())
+                return Enum.valueOf(cls, val);
+
             return val;
         }
 
@@ -360,10 +363,11 @@ public class VisorGatewayTask implements ComputeTask<Object[], Object> {
                             // Length of arguments that required to constructor by influence of nested complex objects.
                             int needArgs = args;
 
-                            for (Class type: types)
+                            for (Class type: types) {
                                 // When constructor required specified types increase length of required arguments.
                                 if (TYPE_ARG_LENGTH.containsKey(type))
                                     needArgs += TYPE_ARG_LENGTH.get(type);
+                            }
 
                             if (needArgs == beanArgsCnt) {
                                 Object[] initArgs = new Object[args];

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
index 07ba214..1754cc8 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
@@ -54,6 +54,7 @@ import org.apache.ignite.plugin.extensions.communication.MessageWriter;
 import org.apache.ignite.plugin.security.SecuritySubject;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.spi.discovery.DiscoveryDataBag;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK;
@@ -899,6 +900,11 @@ public abstract class IgniteSpiAdapter implements IgniteSpi {
         }
 
         /** {@inheritDoc} */
+        @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode node, DiscoveryDataBag discoData) {
+            return null;
+        }
+
+        /** {@inheritDoc} */
         @Override public Collection<SecuritySubject> authenticatedSubjects() {
             return Collections.emptyList();
         }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
index 96b3e61..108c4d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
@@ -31,6 +31,7 @@ import org.apache.ignite.lang.IgniteBiPredicate;
 import org.apache.ignite.plugin.extensions.communication.MessageFactory;
 import org.apache.ignite.plugin.extensions.communication.MessageFormatter;
 import org.apache.ignite.plugin.security.SecuritySubject;
+import org.apache.ignite.spi.discovery.DiscoveryDataBag;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -296,6 +297,12 @@ public interface IgniteSpiContext {
     @Nullable public IgniteNodeValidationResult validateNode(ClusterNode node);
 
     /**
+     * @param node Node.
+     * @param discoData Disco data.
+     */
+    @Nullable public IgniteNodeValidationResult validateNode(ClusterNode node, DiscoveryDataBag discoData);
+
+    /**
      * Gets collection of authenticated subjects together with their permissions.
      *
      * @return Collection of authenticated subjects.

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index b9f381b..6f79720 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -3574,36 +3574,43 @@ class ServerImpl extends TcpDiscoveryImpl {
                     }
                 }
 
-                final IgniteNodeValidationResult err = spi.getSpiContext().validateNode(node);
+                IgniteNodeValidationResult err;
+
+                err = spi.getSpiContext().validateNode(node);
+
+                if (err == null)
+                    err = spi.getSpiContext().validateNode(node, msg.gridDiscoveryData().unmarshalJoiningNodeData(spi.marshaller(), U.resolveClassLoader(spi.ignite().configuration()), false, log));
 
                 if (err != null) {
+                    final IgniteNodeValidationResult err0 = err;
+
                     if (log.isDebugEnabled())
                         log.debug("Node validation failed [res=" + err + ", node=" + node + ']');
 
                     utilityPool.execute(
                         new Runnable() {
                             @Override public void run() {
-                                boolean ping = node.id().equals(err.nodeId()) ? pingNode(node) : pingNode(err.nodeId());
+                                boolean ping = node.id().equals(err0.nodeId()) ? pingNode(node) : pingNode(err0.nodeId());
 
                                 if (!ping) {
                                     if (log.isDebugEnabled())
                                         log.debug("Conflicting node has already left, need to wait for event. " +
                                             "Will ignore join request for now since it will be recent [req=" + msg +
-                                            ", err=" + err.message() + ']');
+                                            ", err=" + err0.message() + ']');
 
                                     // Ignore join request.
                                     return;
                                 }
 
-                                LT.warn(log, err.message());
+                                LT.warn(log, err0.message());
 
                                 // Always output in debug.
                                 if (log.isDebugEnabled())
-                                    log.debug(err.message());
+                                    log.debug(err0.message());
 
                                 try {
                                     trySendMessageDirectly(node,
-                                        new TcpDiscoveryCheckFailedMessage(err.nodeId(), err.sendMessage()));
+                                        new TcpDiscoveryCheckFailedMessage(err0.nodeId(), err0.sendMessage()));
                                 }
                                 catch (IgniteSpiException e) {
                                     if (log.isDebugEnabled())

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 3b83b2e..713bbab 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -972,17 +972,23 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi {
 
             initAddresses();
 
-            final Serializable cfgId = ignite.configuration().getConsistentId();
+            IgniteConfiguration cfg = ignite.configuration();
+
+            final Serializable cfgId = cfg.getConsistentId();
 
             if (cfgId == null) {
-                final List<String> sortedAddrs = new ArrayList<>(addrs.get1());
+                if (cfg.isClientMode() == Boolean.TRUE)
+                    consistentId = cfg.getNodeId();
+                else {
+                    List<String> sortedAddrs = new ArrayList<>(addrs.get1());
 
-                Collections.sort(sortedAddrs);
+                    Collections.sort(sortedAddrs);
 
-                if (getBoolean(IGNITE_CONSISTENT_ID_BY_HOST_WITHOUT_PORT))
-                    consistentId = U.consistentId(sortedAddrs);
-                else
-                    consistentId = U.consistentId(sortedAddrs, impl.boundPort());
+                    if (getBoolean(IGNITE_CONSISTENT_ID_BY_HOST_WITHOUT_PORT))
+                        consistentId = U.consistentId(sortedAddrs);
+                    else
+                        consistentId = U.consistentId(sortedAddrs, impl.boundPort());
+                }
             }
             else
                 consistentId = cfgId;

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/DiscoveryDataPacket.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/DiscoveryDataPacket.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/DiscoveryDataPacket.java
index 1134de6..5f70f3c 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/DiscoveryDataPacket.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/DiscoveryDataPacket.java
@@ -48,6 +48,9 @@ public class DiscoveryDataPacket implements Serializable {
     private Map<Integer, byte[]> joiningNodeData = new HashMap<>();
 
     /** */
+    private transient Map<Integer, Serializable> unmarshalledJoiningNodeData;
+
+    /** */
     private Map<Integer, byte[]> commonData = new HashMap<>();
 
     /** */
@@ -154,14 +157,14 @@ public class DiscoveryDataPacket implements Serializable {
         DiscoveryDataBag dataBag = new DiscoveryDataBag(joiningNodeId);
 
         if (joiningNodeData != null && !joiningNodeData.isEmpty()) {
-            Map<Integer, Serializable> unmarshJoiningNodeData = unmarshalData(
+            unmarshalledJoiningNodeData = unmarshalData(
                     joiningNodeData,
                     marsh,
                     clsLdr,
                     clientNode,
                     log);
 
-            dataBag.joiningNodeData(unmarshJoiningNodeData);
+            dataBag.joiningNodeData(unmarshalledJoiningNodeData);
         }
 
         return dataBag;
@@ -340,6 +343,11 @@ public class DiscoveryDataPacket implements Serializable {
      * (e.g. on nodes prior in cluster to the one where this method is called).
      */
     public DiscoveryDataBag bagForDataCollection() {
-        return new DiscoveryDataBag(joiningNodeId, commonData.keySet());
+        DiscoveryDataBag dataBag = new DiscoveryDataBag(joiningNodeId, commonData.keySet());
+
+        if (unmarshalledJoiningNodeData != null)
+            dataBag.joiningNodeData(unmarshalledJoiningNodeData);
+
+        return dataBag;
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
index 38c2a1b..01534f7 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
@@ -623,8 +623,6 @@ public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements Cluste
 
         Object consistentIdAttr = attrs.get(ATTR_NODE_CONSISTENT_ID);
 
-        consistentId = consistentIdAttr != null ? consistentIdAttr : U.consistentId(addrs, discPort);
-
         // Cluster metrics
         byte[] mtr = U.readByteArray(in);
 
@@ -648,6 +646,11 @@ public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements Cluste
         intOrder = in.readLong();
         ver = (IgniteProductVersion)in.readObject();
         clientRouterNodeId = U.readUuid(in);
+
+        if (isClient())
+            consistentId = consistentIdAttr != null ? consistentIdAttr : id;
+        else
+            consistentId = consistentIdAttr != null ? consistentIdAttr : U.consistentId(addrs, discPort);
     }
 
     /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/resources/META-INF/classnames.properties
----------------------------------------------------------------------
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index c27681e..cb70857 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -209,7 +209,6 @@ org.apache.ignite.internal.GridKernalGatewayImpl
 org.apache.ignite.internal.GridKernalState
 org.apache.ignite.internal.GridLoggerProxy
 org.apache.ignite.internal.GridMessageListenHandler
-org.apache.ignite.internal.GridNodeOrderComparator
 org.apache.ignite.internal.GridTaskCancelRequest
 org.apache.ignite.internal.GridTaskNameHashKey
 org.apache.ignite.internal.GridTaskSessionRequest
@@ -302,6 +301,8 @@ org.apache.ignite.internal.cluster.IgniteClusterImpl
 org.apache.ignite.internal.cluster.IgniteClusterImpl$1
 org.apache.ignite.internal.cluster.IgniteKillTask
 org.apache.ignite.internal.cluster.IgniteKillTask$IgniteKillJob
+org.apache.ignite.internal.cluster.NodeOrderComparator
+org.apache.ignite.internal.cluster.NodeOrderLegacyComparator
 org.apache.ignite.internal.compute.ComputeTaskCancelledCheckedException
 org.apache.ignite.internal.compute.ComputeTaskTimeoutCheckedException
 org.apache.ignite.internal.direct.DirectMessageReader$1
@@ -311,6 +312,14 @@ org.apache.ignite.internal.executor.GridExecutorService
 org.apache.ignite.internal.executor.GridExecutorService$1
 org.apache.ignite.internal.executor.GridExecutorService$TaskTerminateListener
 org.apache.ignite.internal.igfs.common.IgfsIpcCommand
+org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl
+org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$1
+org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$BooleanProperty
+org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$ConnectionProperty
+org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$IntegerProperty
+org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$NumberProperty
+org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$PropertyValidator
+org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$StringProperty
 org.apache.ignite.internal.jdbc2.JdbcBatchUpdateTask
 org.apache.ignite.internal.jdbc2.JdbcConnection$JdbcConnectionValidationTask
 org.apache.ignite.internal.jdbc2.JdbcDatabaseMetadata$UpdateMetadataTask
@@ -335,11 +344,14 @@ org.apache.ignite.internal.managers.deployment.GridDeploymentResponse
 org.apache.ignite.internal.managers.discovery.CustomMessageWrapper
 org.apache.ignite.internal.managers.discovery.DiscoCache$1
 org.apache.ignite.internal.managers.discovery.DiscoCache$2
+org.apache.ignite.internal.managers.discovery.DiscoCache$3
 org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage
 org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$1
 org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$2
 org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$4$1
 org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$6
+org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$8
+org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$9
 org.apache.ignite.internal.managers.discovery.GridLocalMetrics
 org.apache.ignite.internal.managers.eventstorage.GridEventStorageMessage
 org.apache.ignite.internal.managers.indexing.GridIndexingManager$1
@@ -351,6 +363,7 @@ org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl$Segment
 org.apache.ignite.internal.pagemem.wal.StorageException
 org.apache.ignite.internal.pagemem.wal.WALIterator
 org.apache.ignite.internal.pagemem.wal.WALPointer
+org.apache.ignite.internal.pagemem.wal.record.ExchangeRecord$Type
 org.apache.ignite.internal.pagemem.wal.record.WALRecord$RecordType
 org.apache.ignite.internal.pagemem.wal.record.delta.DeltaApplicationException
 org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion
@@ -414,6 +427,7 @@ org.apache.ignite.internal.processors.cache.CacheObjectImpl
 org.apache.ignite.internal.processors.cache.CacheOperationContext
 org.apache.ignite.internal.processors.cache.CacheOperationFilter
 org.apache.ignite.internal.processors.cache.CachePartialUpdateCheckedException
+org.apache.ignite.internal.processors.cache.CacheStatisticsModeChangeMessage
 org.apache.ignite.internal.processors.cache.CacheStoppedException
 org.apache.ignite.internal.processors.cache.CacheStorePartialUpdateException
 org.apache.ignite.internal.processors.cache.CacheType
@@ -436,9 +450,9 @@ org.apache.ignite.internal.processors.cache.GridCacheAdapter$12
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$13
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$14
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$15
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$16$1
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$16
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$17
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$18
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$18$1
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$2
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$26$1
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$28
@@ -529,10 +543,12 @@ org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$Ex
 org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$MessageHandler
 org.apache.ignite.internal.processors.cache.GridCacheProcessor$1
 org.apache.ignite.internal.processors.cache.GridCacheProcessor$10
+org.apache.ignite.internal.processors.cache.GridCacheProcessor$11
+org.apache.ignite.internal.processors.cache.GridCacheProcessor$2
 org.apache.ignite.internal.processors.cache.GridCacheProcessor$3
 org.apache.ignite.internal.processors.cache.GridCacheProcessor$4
 org.apache.ignite.internal.processors.cache.GridCacheProcessor$5
-org.apache.ignite.internal.processors.cache.GridCacheProcessor$7
+org.apache.ignite.internal.processors.cache.GridCacheProcessor$6
 org.apache.ignite.internal.processors.cache.GridCacheProcessor$8
 org.apache.ignite.internal.processors.cache.GridCacheProcessor$9
 org.apache.ignite.internal.processors.cache.GridCacheProcessor$LocalAffinityFunction
@@ -555,6 +571,7 @@ org.apache.ignite.internal.processors.cache.GridCacheUtils$18
 org.apache.ignite.internal.processors.cache.GridCacheUtils$19
 org.apache.ignite.internal.processors.cache.GridCacheUtils$2
 org.apache.ignite.internal.processors.cache.GridCacheUtils$20
+org.apache.ignite.internal.processors.cache.GridCacheUtils$22
 org.apache.ignite.internal.processors.cache.GridCacheUtils$3
 org.apache.ignite.internal.processors.cache.GridCacheUtils$4
 org.apache.ignite.internal.processors.cache.GridCacheUtils$5
@@ -562,6 +579,7 @@ org.apache.ignite.internal.processors.cache.GridCacheUtils$6
 org.apache.ignite.internal.processors.cache.GridCacheUtils$7
 org.apache.ignite.internal.processors.cache.GridCacheUtils$8
 org.apache.ignite.internal.processors.cache.GridCacheUtils$9
+org.apache.ignite.internal.processors.cache.GridCacheUtils$BackupPostProcessingClosure
 org.apache.ignite.internal.processors.cache.GridCacheValueCollection
 org.apache.ignite.internal.processors.cache.GridCacheValueCollection$1
 org.apache.ignite.internal.processors.cache.GridChangeGlobalStateMessageResponse
@@ -898,12 +916,13 @@ org.apache.ignite.internal.processors.cache.local.atomic.GridLocalAtomicCache$5
 org.apache.ignite.internal.processors.cache.local.atomic.GridLocalAtomicCache$8
 org.apache.ignite.internal.processors.cache.local.atomic.GridLocalAtomicCache$9
 org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter$RowData
-org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$6
-org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$9
+org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$11
+org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$7
 org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$CheckpointEntryType
 org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager$RebalanceIteratorAdapter
 org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager$1
 org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory
+org.apache.ignite.internal.processors.cache.persistence.file.FileDownloader$1
 org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory
 org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory
 org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl$Segment
@@ -914,10 +933,16 @@ org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$DestroyBa
 org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Result
 org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO$EntryPart
 org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator
+org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator$StartSeekingFilter
 org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer
 org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager$FileArchiver$1
+org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager$FileArchiver$2
+org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager$FileCompressor$1
 org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager$RecordsIterator
 org.apache.ignite.internal.processors.cache.persistence.wal.SegmentEofException
+org.apache.ignite.internal.processors.cache.persistence.wal.SegmentedRingByteBuffer$BufferMode
+org.apache.ignite.internal.processors.cache.persistence.wal.SingleSegmentLogicalRecordsIterator
+org.apache.ignite.internal.processors.cache.persistence.wal.SingleSegmentLogicalRecordsIterator$LogicalRecordsFilter
 org.apache.ignite.internal.processors.cache.persistence.wal.WalSegmentTailReachedException
 org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException
 org.apache.ignite.internal.processors.cache.persistence.wal.reader.StandaloneWalRecordsIterator
@@ -1049,6 +1074,15 @@ org.apache.ignite.internal.processors.cache.transactions.TxLock
 org.apache.ignite.internal.processors.cache.transactions.TxLockList
 org.apache.ignite.internal.processors.cache.transactions.TxLocksRequest
 org.apache.ignite.internal.processors.cache.transactions.TxLocksResponse
+org.apache.ignite.internal.processors.cache.verify.CollectConflictPartitionKeysTask
+org.apache.ignite.internal.processors.cache.verify.CollectConflictPartitionKeysTask$CollectPartitionEntryHashesJob
+org.apache.ignite.internal.processors.cache.verify.PartitionEntryHashRecord
+org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord
+org.apache.ignite.internal.processors.cache.verify.PartitionKey
+org.apache.ignite.internal.processors.cache.verify.RetrieveConflictPartitionValuesTask
+org.apache.ignite.internal.processors.cache.verify.RetrieveConflictPartitionValuesTask$RetrieveConflictValuesJob
+org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTask
+org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTask$VerifyBackupPartitionsJob
 org.apache.ignite.internal.processors.cache.version.GridCacheRawVersionedEntry
 org.apache.ignite.internal.processors.cache.version.GridCacheVersion
 org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext$State
@@ -1075,6 +1109,10 @@ org.apache.ignite.internal.processors.closure.GridClosureProcessor$T8
 org.apache.ignite.internal.processors.closure.GridClosureProcessor$T9
 org.apache.ignite.internal.processors.closure.GridClosureProcessor$TaskNoReduceAdapter
 org.apache.ignite.internal.processors.closure.GridPeerDeployAwareTaskAdapter
+org.apache.ignite.internal.processors.cluster.BaselineTopology
+org.apache.ignite.internal.processors.cluster.BaselineTopologyHistory
+org.apache.ignite.internal.processors.cluster.BaselineTopologyHistoryItem
+org.apache.ignite.internal.processors.cluster.BranchingPointType
 org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage
 org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage
 org.apache.ignite.internal.processors.cluster.ClusterProcessor$3
@@ -1086,6 +1124,7 @@ org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$3
 org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$4
 org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$5
 org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$7
+org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$BaselineStateAndHistoryData
 org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$CheckGlobalStateComputeRequest
 org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$ClientChangeGlobalStateComputeRequest
 org.apache.ignite.internal.processors.continuous.AbstractContinuousMessage
@@ -1388,6 +1427,7 @@ org.apache.ignite.internal.processors.query.schema.message.SchemaProposeDiscover
 org.apache.ignite.internal.processors.query.schema.operation.SchemaAbstractAlterTableOperation
 org.apache.ignite.internal.processors.query.schema.operation.SchemaAbstractOperation
 org.apache.ignite.internal.processors.query.schema.operation.SchemaAlterTableAddColumnOperation
+org.apache.ignite.internal.processors.query.schema.operation.SchemaAlterTableDropColumnOperation
 org.apache.ignite.internal.processors.query.schema.operation.SchemaIndexAbstractOperation
 org.apache.ignite.internal.processors.query.schema.operation.SchemaIndexCreateOperation
 org.apache.ignite.internal.processors.query.schema.operation.SchemaIndexDropOperation
@@ -1621,6 +1661,7 @@ org.apache.ignite.internal.util.lang.GridAbsClosure
 org.apache.ignite.internal.util.lang.GridAbsClosureX
 org.apache.ignite.internal.util.lang.GridCloseableIterator
 org.apache.ignite.internal.util.lang.GridClosureException
+org.apache.ignite.internal.util.lang.GridFunc$1
 org.apache.ignite.internal.util.lang.GridIterable
 org.apache.ignite.internal.util.lang.GridIterableAdapter
 org.apache.ignite.internal.util.lang.GridIterableAdapter$IteratorWrapper
@@ -1727,6 +1768,7 @@ org.apache.ignite.internal.util.snaptree.EpochNode
 org.apache.ignite.internal.util.snaptree.EpochNode$Child
 org.apache.ignite.internal.util.snaptree.SnapTreeMap
 org.apache.ignite.internal.util.snaptree.SnapTreeMap$SubMap
+org.apache.ignite.internal.util.tostring.SBLimitedLength
 org.apache.ignite.internal.util.typedef.C1
 org.apache.ignite.internal.util.typedef.C2
 org.apache.ignite.internal.util.typedef.CA
@@ -1767,6 +1809,12 @@ org.apache.ignite.internal.visor.VisorJob
 org.apache.ignite.internal.visor.VisorMultiNodeTask
 org.apache.ignite.internal.visor.VisorOneNodeTask
 org.apache.ignite.internal.visor.VisorTaskArgument
+org.apache.ignite.internal.visor.baseline.VisorBaselineNode
+org.apache.ignite.internal.visor.baseline.VisorBaselineOperation
+org.apache.ignite.internal.visor.baseline.VisorBaselineTask
+org.apache.ignite.internal.visor.baseline.VisorBaselineTask$VisorBaselineJob
+org.apache.ignite.internal.visor.baseline.VisorBaselineTaskArg
+org.apache.ignite.internal.visor.baseline.VisorBaselineTaskResult
 org.apache.ignite.internal.visor.binary.VisorBinaryMetadata
 org.apache.ignite.internal.visor.binary.VisorBinaryMetadataCollectorTask
 org.apache.ignite.internal.visor.binary.VisorBinaryMetadataCollectorTask$VisorBinaryCollectMetadataJob
@@ -1802,7 +1850,7 @@ org.apache.ignite.internal.visor.cache.VisorCacheMetricsCollectorTask
 org.apache.ignite.internal.visor.cache.VisorCacheMetricsCollectorTask$VisorCacheMetricsCollectorJob
 org.apache.ignite.internal.visor.cache.VisorCacheMetricsCollectorTaskArg
 org.apache.ignite.internal.visor.cache.VisorCacheModifyTask
-org.apache.ignite.internal.visor.cache.VisorCacheModifyTask$VisorCacheClearJob
+org.apache.ignite.internal.visor.cache.VisorCacheModifyTask$VisorCacheModifyJob
 org.apache.ignite.internal.visor.cache.VisorCacheModifyTaskArg
 org.apache.ignite.internal.visor.cache.VisorCacheModifyTaskResult
 org.apache.ignite.internal.visor.cache.VisorCacheNearConfiguration
@@ -2057,23 +2105,23 @@ org.apache.ignite.spi.checkpoint.sharedfs.SharedFsCheckpointData
 org.apache.ignite.spi.collision.jobstealing.JobStealingRequest
 org.apache.ignite.spi.collision.priorityqueue.PriorityQueueCollisionSpi$PriorityGridCollisionJobContextComparator
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$1
+org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$10
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$11
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$12
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$1
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$2
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosure
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosure$1
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosureNew
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosureNew$1
+org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$3
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$4
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$5
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeClosure
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeException
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeMessage
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeMessage2
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeTimeoutException
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$NodeIdMessage
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$RecoveryLastReceivedMessage
+org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage
+org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage2
+org.apache.ignite.spi.communication.tcp.messages.NodeIdMessage
+org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage
 org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage
 org.apache.ignite.spi.discovery.tcp.ClientImpl$State
 org.apache.ignite.spi.discovery.tcp.ServerImpl$IpFinderCleaner$1

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
index 1c10bf1..518e674 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
@@ -1024,7 +1024,11 @@ public class IgniteClientReconnectCacheTest extends IgniteClientReconnectAbstrac
         List<IgniteCache> caches = new ArrayList<>();
 
         for (int i = 0; i < CLIENTS; i++) {
-            Ignite client = startGrid(SRV_CNT + i);
+            int g = SRV_CNT + i;
+
+            IgniteEx client = startGrid(g);
+
+            info(">>>>> Started client: " + g);
 
             addListener(client, disconnectLatch, reconnectLatch);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractRemoveFailureTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractRemoveFailureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractRemoveFailureTest.java
index faa19dc..11f19b2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractRemoveFailureTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractRemoveFailureTest.java
@@ -156,7 +156,7 @@ public abstract class GridCacheAbstractRemoveFailureTest extends GridCommonAbstr
      * @throws Exception If failed.
      */
     public void testPutAndRemove() throws Exception {
-        putAndRemove(DUR, null, null);
+        putAndRemove(duration(), null, null);
     }
 
     /**
@@ -166,7 +166,7 @@ public abstract class GridCacheAbstractRemoveFailureTest extends GridCommonAbstr
         if (atomicityMode() != CacheAtomicityMode.TRANSACTIONAL)
             return;
 
-        putAndRemove(30_000, PESSIMISTIC, REPEATABLE_READ);
+        putAndRemove(duration(), PESSIMISTIC, REPEATABLE_READ);
     }
 
     /**
@@ -176,7 +176,12 @@ public abstract class GridCacheAbstractRemoveFailureTest extends GridCommonAbstr
         if (atomicityMode() != CacheAtomicityMode.TRANSACTIONAL)
             return;
 
-        putAndRemove(30_000, OPTIMISTIC, SERIALIZABLE);
+        putAndRemove(duration(), OPTIMISTIC, SERIALIZABLE);
+    }
+
+    /** */
+    protected long duration() {
+        return DUR;
     }
 
     /**
@@ -314,29 +319,7 @@ public abstract class GridCacheAbstractRemoveFailureTest extends GridCommonAbstr
             }
         });
 
-        IgniteInternalFuture<?> killFut = GridTestUtils.runAsync(new Callable<Void>() {
-            @Override public Void call() throws Exception {
-                Thread.currentThread().setName("restart-thread");
-
-                while (!stop.get()) {
-                    U.sleep(random(KILL_DELAY.get1(), KILL_DELAY.get2()));
-
-                    killAndRestart(stop);
-
-                    CyclicBarrier barrier = cmp.get();
-
-                    if (barrier != null) {
-                        log.info("Wait data check.");
-
-                        barrier.await(60_000, TimeUnit.MILLISECONDS);
-
-                        log.info("Finished wait data check.");
-                    }
-                }
-
-                return null;
-            }
-        });
+        IgniteInternalFuture killFut = createAndRunConcurrentAction(stop, cmp);
 
         try {
             long stopTime = duration + U.currentTimeMillis() ;
@@ -408,25 +391,50 @@ public abstract class GridCacheAbstractRemoveFailureTest extends GridCommonAbstr
         log.info("Test finished. Update errors: " + errCntr.get());
     }
 
+    /** */
+    protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean stop, final AtomicReference<CyclicBarrier> cmp) {
+        return GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                Thread.currentThread().setName("restart-thread");
+
+                while (!stop.get()) {
+                    U.sleep(random(KILL_DELAY.get1(), KILL_DELAY.get2()));
+
+                    killAndRestart(stop, random(1, GRID_CNT + 1));
+
+                    CyclicBarrier barrier = cmp.get();
+
+                    if (barrier != null) {
+                        log.info("Wait data check.");
+
+                        barrier.await(60_000, TimeUnit.MILLISECONDS);
+
+                        log.info("Finished wait data check.");
+                    }
+                }
+
+                return null;
+            }
+        });
+    }
+
     /**
      * @param stop Stop flag.
      * @throws Exception If failed.
      */
-    private void killAndRestart(AtomicBoolean stop) throws Exception {
+    protected void killAndRestart(AtomicBoolean stop, int nodeIdx) throws Exception {
         if (stop.get())
             return;
 
-        int idx = random(1, GRID_CNT + 1);
-
-        log.info("Killing node " + idx);
+        log.info("Killing node " + nodeIdx);
 
-        stopGrid(idx);
+        stopGrid(nodeIdx);
 
         U.sleep(random(START_DELAY.get1(), START_DELAY.get2()));
 
-        log.info("Restarting node " + idx);
+        log.info("Restarting node " + nodeIdx);
 
-        startGrid(idx);
+        startGrid(nodeIdx);
 
         if (stop.get())
             return;
@@ -475,7 +483,7 @@ public abstract class GridCacheAbstractRemoveFailureTest extends GridCommonAbstr
      * @param max Max possible value (exclusive).
      * @return Random value.
      */
-    private static int random(int min, int max) {
+    protected static int random(int min, int max) {
         if (max == min)
             return max;
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
index 81aadb9..df93ae4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
@@ -34,7 +34,6 @@ import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
@@ -113,7 +112,7 @@ public abstract class GridCacheAbstractSelfTest extends GridCommonAbstractTest {
      *
      * @throws IgniteCheckedException If failed.
      */
-    void initStoreStrategy() throws IgniteCheckedException {
+    protected void initStoreStrategy() throws IgniteCheckedException {
         if (storeStgy == null)
             storeStgy = isMultiJvm() ? new H2CacheStoreStrategy() : new MapCacheStoreStrategy();
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
index 3d1b014..e40700d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
@@ -459,7 +459,7 @@ public class GridCacheTestEntryEx extends GridMetadataAwareAdapter implements Gr
         @Nullable GridCacheVersion dhtVer,
         @Nullable Long updateCntr)
         throws IgniteCheckedException, GridCacheEntryRemovedException {
-        return new GridCacheUpdateTxResult(true, rawPut(val, ttl));
+        return new GridCacheUpdateTxResult(true, rawPut(val, ttl), null);
     }
 
     /** {@inheritDoc} */
@@ -545,7 +545,7 @@ public class GridCacheTestEntryEx extends GridMetadataAwareAdapter implements Gr
 
         val = null;
 
-        return new GridCacheUpdateTxResult(true, old);
+        return new GridCacheUpdateTxResult(true, old, null);
     }
 
     /** @inheritDoc */

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index 1827c65..267cb18 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -26,6 +26,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -34,6 +35,7 @@ import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.IgniteClientReconnectAbstractTest;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.TestRecordingCommunicationSpi;
@@ -784,7 +786,7 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
         startWithCaches1(SRVS, CLIENTS);
 
         final Ignite srv = ignite(0);
-        Ignite client = ignite(SRVS);
+        IgniteEx client = grid(SRVS);
 
         if (persistenceEnabled())
             ignite(0).active(true);
@@ -822,12 +824,11 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
             }
         });
 
-        checkCache(client, CACHE_NAME_PREFIX + 0, false);
-
         if (transition) {
             assertFalse(stateFut.get().isDone());
 
-            assertFalse(client.active());
+            // Public API method would block forever because we blocked the exchange message.
+            assertFalse(client.context().state().publicApiActiveState(false));
 
             spi1.waitForBlocked();
 
@@ -889,7 +890,7 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
         startWithCaches1(SRVS, CLIENTS);
 
         final Ignite srv = ignite(0);
-        Ignite client = ignite(SRVS);
+        IgniteEx client = grid(SRVS);
 
         checkNoCaches(SRVS + CLIENTS);
 
@@ -922,12 +923,10 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
             }
         });
 
-        checkCache(client, CACHE_NAME_PREFIX + 0, !transition);
-
         if (transition) {
             assertFalse(stateFut.get().isDone());
 
-            assertFalse(client.active());
+            assertTrue(client.context().state().clusterState().transition());
 
             spi1.waitForBlocked();
 
@@ -1140,70 +1139,6 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
     }
 
     /**
-     * @throws Exception If failed.
-     */
-    public void testActivateFailover3() throws Exception {
-        stateChangeFailover3(true);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testDeactivateFailover3() throws Exception {
-        stateChangeFailover3(false);
-    }
-
-    /**
-     * @param activate If {@code true} tests activation, otherwise deactivation.
-     * @throws Exception If failed.
-     */
-    private void stateChangeFailover3(boolean activate) throws Exception {
-        testDiscoSpi = true;
-
-        startNodesAndBlockStatusChange(4, 0, 0, !activate);
-
-        client = false;
-
-        IgniteInternalFuture startFut1 = GridTestUtils.runAsync(new Callable() {
-            @Override public Object call() throws Exception {
-                startGrid(4);
-
-                return null;
-            }
-        }, "start-node1");
-
-        IgniteInternalFuture startFut2 = GridTestUtils.runAsync(new Callable() {
-            @Override public Object call() throws Exception {
-                startGrid(5);
-
-                return null;
-            }
-        }, "start-node2");
-
-        U.sleep(1000);
-
-        // Stop all nodes participating in state change and not allow last node to finish exchange.
-        for (int i = 0; i < 4; i++)
-            ((TestTcpDiscoverySpi)ignite(i).configuration().getDiscoverySpi()).simulateNodeFailure();
-
-        for (int i = 0; i < 4; i++)
-            stopGrid(getTestIgniteInstanceName(i), true, false);
-
-        startFut1.get();
-        startFut2.get();
-
-        assertFalse(ignite(4).active());
-        assertFalse(ignite(5).active());
-
-        ignite(4).active(true);
-
-        for (int i = 0; i < 4; i++)
-            startGrid(i);
-
-        checkCaches1(6);
-    }
-
-    /**
      * @param exp If {@code true} there should be recorded messages.
      */
     private void checkRecordedMessages(boolean exp) {
@@ -1274,7 +1209,11 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
      * @param node Node.
      * @param exp {@code True} if expect that cache is started on node.
      */
-    void checkCache(Ignite node, String cacheName, boolean exp) {
+    void checkCache(Ignite node, String cacheName, boolean exp) throws IgniteCheckedException {
+        ((IgniteEx)node).context().cache().context().exchange().lastTopologyFuture().get();
+
+        ((IgniteEx)node).context().state().publicApiActiveState(true);
+
         GridCacheAdapter cache = ((IgniteKernal)node).context().cache().internalCache(cacheName);
 
         if (exp)
@@ -1288,6 +1227,8 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
      */
     final void checkNoCaches(int nodes) {
         for (int i = 0; i < nodes; i++) {
+            grid(i).context().state().publicApiActiveState(true);
+
             GridCacheProcessor cache = ((IgniteKernal)ignite(i)).context().cache();
 
             assertTrue(cache.caches().isEmpty());

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java
index 15e67b2..76078bc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java
@@ -20,10 +20,14 @@ package org.apache.ignite.internal.processors.cache;
 import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.testframework.GridTestUtils;
@@ -51,6 +55,11 @@ public class IgniteClusterActivateDeactivateTestWithPersistence extends IgniteCl
         GridTestUtils.deleteDbFiles();
     }
 
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        return super.getConfiguration(igniteInstanceName).setAutoActivationEnabled(false);
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -79,12 +88,8 @@ public class IgniteClusterActivateDeactivateTestWithPersistence extends IgniteCl
         activateCachesRestore(5, true);
     }
 
-    /**
-     * @param srvs Number of server nodes.
-     * @param withNewCaches If {@code true} then after restart has new caches in configuration.
-     * @throws Exception If failed.
-     */
-    private void activateCachesRestore(int srvs, boolean withNewCaches) throws Exception {
+    /** */
+    private Map<Integer, Integer> startGridsAndLoadData(int srvs) throws Exception {
         Ignite srv = startGrids(srvs);
 
         srv.active(true);
@@ -101,6 +106,17 @@ public class IgniteClusterActivateDeactivateTestWithPersistence extends IgniteCl
             }
         }
 
+        return cacheData;
+    }
+
+    /**
+     * @param srvs Number of server nodes.
+     * @param withNewCaches If {@code true} then after restart has new caches in configuration.
+     * @throws Exception If failed.
+     */
+    private void activateCachesRestore(int srvs, boolean withNewCaches) throws Exception {
+        Map<Integer, Integer> cacheData = startGridsAndLoadData(srvs);
+
         stopAllGrids();
 
         for (int i = 0; i < srvs; i++) {
@@ -110,7 +126,7 @@ public class IgniteClusterActivateDeactivateTestWithPersistence extends IgniteCl
             startGrid(i);
         }
 
-        srv = ignite(0);
+        Ignite srv = ignite(0);
 
         checkNoCaches(srvs);
 
@@ -160,6 +176,47 @@ public class IgniteClusterActivateDeactivateTestWithPersistence extends IgniteCl
     }
 
     /**
+     * @see <a href="https://issues.apache.org/jira/browse/IGNITE-7330">IGNITE-7330</a> for more information about context of the test
+     */
+    public void testClientJoinsWhenActivationIsInProgress() throws Exception {
+        startGridsAndLoadData(5);
+
+        stopAllGrids();
+
+        Ignite srv = startGrids(5);
+
+        final CountDownLatch clientStartLatch = new CountDownLatch(1);
+
+        IgniteInternalFuture clStartFut = GridTestUtils.runAsync(new Runnable() {
+            @Override public void run() {
+                try {
+                    clientStartLatch.await();
+
+                    Thread.sleep(10);
+
+                    client = true;
+
+                    Ignite cl = startGrid("client0");
+
+                    IgniteCache<Object, Object> atomicCache = cl.cache(CACHE_NAME_PREFIX + '0');
+                    IgniteCache<Object, Object> txCache = cl.cache(CACHE_NAME_PREFIX + '1');
+
+                    assertEquals(100, atomicCache.size());
+                    assertEquals(100, txCache.size());
+                }
+                catch (Exception e) {
+                    log.error("Error occurred", e);
+                }
+            }
+        }, "client-starter-thread");
+
+        clientStartLatch.countDown();
+        srv.active(true);
+
+        clStartFut.get();
+    }
+
+    /**
      * Checks that persistent caches are present with actual data and volatile caches are missing.
      *
      * @param cacheData Cache data.
@@ -214,8 +271,4 @@ public class IgniteClusterActivateDeactivateTestWithPersistence extends IgniteCl
 
         checkNoCaches(SRVS);
     }
-
-    @Override public void testActivateFailover3() throws Exception {
-        super.testActivateFailover3();
-    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
new file mode 100644
index 0000000..5fbd752
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
@@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.PartitionLossPolicy;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.PA;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.cache.PartitionLossPolicy.READ_ONLY_SAFE;
+
+/**
+ *
+ */
+public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
+    /** */
+    private static final String CACHE_NAME = "cache";
+
+    /** */
+    private boolean client;
+
+    /** */
+    private static final int NODE_COUNT = 4;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+
+        client = false;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration().setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setPersistenceEnabled(true)
+                    .setMaxSize(100 * 1024 * 1024)
+                    .setInitialSize(100 * 1024 * 1024)
+            )
+            .setWalMode(WALMode.LOG_ONLY)
+        );
+
+        if (client)
+            cfg.setClientMode(true);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTopologyChangesWithFixedBaseline() throws Exception {
+        startGrids(NODE_COUNT);
+
+        IgniteEx ignite = grid(0);
+
+        ignite.active(true);
+
+        awaitPartitionMapExchange();
+
+        Map<ClusterNode, Ignite> nodes = new HashMap<>();
+
+        for (int i = 0; i < NODE_COUNT; i++) {
+            Ignite ig = grid(i);
+
+            nodes.put(ig.cluster().localNode(), ig);
+        }
+
+        IgniteCache<Integer, Integer> cache =
+            ignite.createCache(
+                new CacheConfiguration<Integer, Integer>()
+                    .setName(CACHE_NAME)
+                    .setCacheMode(PARTITIONED)
+                    .setBackups(1)
+                    .setPartitionLossPolicy(READ_ONLY_SAFE)
+            );
+
+        int key = -1;
+
+        for (int k = 0; k < 100_000; k++) {
+            if (!ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(k).contains(ignite.localNode())) {
+                key = k;
+                break;
+            }
+        }
+
+        assert key >= 0;
+
+        int part = ignite.affinity(CACHE_NAME).partition(key);
+
+        Collection<ClusterNode> initialMapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+        assert initialMapping.size() == 2 : initialMapping;
+
+        ignite.cluster().setBaselineTopology(baselineNodes(nodes.keySet()));
+
+        awaitPartitionMapExchange();
+
+        cache.put(key, 1);
+
+        Collection<ClusterNode> mapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+        assert initialMapping.size() == mapping.size() : mapping;
+        assert initialMapping.containsAll(mapping) : mapping;
+
+        IgniteEx newIgnite = startGrid(4);
+
+        awaitPartitionMapExchange();
+
+        mapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+        assert initialMapping.size() == mapping.size() : mapping;
+        assert initialMapping.containsAll(mapping) : mapping;
+
+        mapping = newIgnite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+        assert initialMapping.size() == mapping.size() : mapping;
+        assert initialMapping.containsAll(mapping) : mapping;
+
+        Set<String> stoppedNodeNames = new HashSet<>();
+
+        ClusterNode node = mapping.iterator().next();
+
+        stoppedNodeNames.add(nodes.get(node).name());
+
+        nodes.get(node).close();
+
+        nodes.remove(node);
+
+        awaitPartitionMapExchange(true, true, null);
+
+        mapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+        assert mapping.size() == 1 : mapping;
+        assert initialMapping.containsAll(mapping);
+
+        node = mapping.iterator().next();
+
+        stoppedNodeNames.add(nodes.get(node).name());
+
+        nodes.get(node).close();
+
+        nodes.remove(node);
+
+        awaitPartitionMapExchange();
+
+        mapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+        assert mapping.isEmpty() : mapping;
+
+        GridDhtPartitionTopology topology = ignite.cachex(CACHE_NAME).context().topology();
+
+        assert topology.lostPartitions().contains(part);
+
+        for (String nodeName : stoppedNodeNames) {
+            startGrid(nodeName);
+        }
+
+        assert ignite.cluster().nodes().size() == NODE_COUNT + 1;
+
+        mapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+        assert initialMapping.size() == mapping.size() : mapping;
+
+        for (ClusterNode n1 : initialMapping) {
+            boolean found = false;
+
+            for (ClusterNode n2 : mapping) {
+                if (n2.consistentId().equals(n1.consistentId())) {
+                    found = true;
+
+                    break;
+                }
+            }
+
+            assert found;
+        }
+
+        ignite.resetLostPartitions(Collections.singleton(CACHE_NAME));
+
+        cache.put(key, 2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBaselineTopologyChangesFromServer() throws Exception {
+        testBaselineTopologyChanges(false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBaselineTopologyChangesFromClient() throws Exception {
+        testBaselineTopologyChanges(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void testBaselineTopologyChanges(boolean fromClient) throws Exception {
+        startGrids(NODE_COUNT);
+
+        IgniteEx ignite;
+
+        if (fromClient) {
+            client = true;
+
+            ignite = startGrid(NODE_COUNT + 10);
+
+            client = false;
+        }
+        else
+            ignite = grid(0);
+
+        ignite.active(true);
+
+        awaitPartitionMapExchange();
+
+        Map<ClusterNode, Ignite> nodes = new HashMap<>();
+
+        for (int i = 0; i < NODE_COUNT; i++) {
+            Ignite ig = grid(i);
+
+            nodes.put(ig.cluster().localNode(), ig);
+        }
+
+        IgniteCache<Integer, Integer> cache =
+            ignite.createCache(
+                new CacheConfiguration<Integer, Integer>()
+                    .setName(CACHE_NAME)
+                    .setCacheMode(PARTITIONED)
+                    .setBackups(1)
+                    .setPartitionLossPolicy(READ_ONLY_SAFE)
+            );
+
+        int key = -1;
+
+        for (int k = 0; k < 100_000; k++) {
+            if (!ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(k).contains(ignite.localNode())) {
+                key = k;
+                break;
+            }
+        }
+
+        assert key >= 0;
+
+        int part = ignite.affinity(CACHE_NAME).partition(key);
+
+        Collection<ClusterNode> initialMapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+        assert initialMapping.size() == 2 : initialMapping;
+
+        ignite.cluster().setBaselineTopology(baselineNodes(nodes.keySet()));
+
+        Set<String> stoppedNodeNames = new HashSet<>();
+
+        ClusterNode node = initialMapping.iterator().next();
+
+        stoppedNodeNames.add(nodes.get(node).name());
+
+        nodes.get(node).close();
+
+        nodes.remove(node);
+
+        awaitPartitionMapExchange();
+
+        Collection<ClusterNode> mapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+        assert mapping.size() == 1 : mapping;
+        assert initialMapping.containsAll(mapping);
+
+        Set<ClusterNode> blt2 = new HashSet<>(ignite.cluster().nodes());
+
+        ignite.cluster().setBaselineTopology(baselineNodes(blt2));
+
+        awaitPartitionMapExchange();
+
+        Collection<ClusterNode> initialMapping2 = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+        assert initialMapping2.size() == 2 : initialMapping2;
+
+        Ignite newIgnite = startGrid(NODE_COUNT);
+
+        awaitPartitionMapExchange();
+
+        mapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+        assert mapping.size() == initialMapping2.size() : mapping;
+        assert mapping.containsAll(initialMapping2);
+
+        assert ignite.affinity(CACHE_NAME).primaryPartitions(newIgnite.cluster().localNode()).length == 0;
+
+        Set<ClusterNode> blt3 = new HashSet<>(ignite.cluster().nodes());
+
+        ignite.cluster().setBaselineTopology(baselineNodes(blt3));
+
+        awaitPartitionMapExchange();
+
+        Collection<ClusterNode> initialMapping3 = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+        assert initialMapping3.size() == 2;
+
+        assert ignite.affinity(CACHE_NAME).primaryPartitions(newIgnite.cluster().localNode()).length > 0;
+
+        newIgnite = startGrid(NODE_COUNT + 1);
+
+        awaitPartitionMapExchange();
+
+        mapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+        assert mapping.size() == initialMapping3.size() : mapping;
+        assert mapping.containsAll(initialMapping3);
+
+        assert ignite.affinity(CACHE_NAME).primaryPartitions(newIgnite.cluster().localNode()).length == 0;
+
+        ignite.cluster().setBaselineTopology(null);
+
+        awaitPartitionMapExchange();
+
+        assert ignite.affinity(CACHE_NAME).primaryPartitions(newIgnite.cluster().localNode()).length > 0;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPrimaryLeft() throws Exception {
+        startGrids(NODE_COUNT);
+
+        IgniteEx ig = grid(0);
+
+        ig.cluster().active(true);
+
+        awaitPartitionMapExchange();
+
+        IgniteCache<Integer, Integer> cache =
+            ig.createCache(
+                new CacheConfiguration<Integer, Integer>()
+                    .setName(CACHE_NAME)
+                    .setCacheMode(PARTITIONED)
+                    .setBackups(1)
+                    .setPartitionLossPolicy(READ_ONLY_SAFE)
+                    .setReadFromBackup(true)
+                    .setWriteSynchronizationMode(FULL_SYNC)
+                    .setRebalanceDelay(-1)
+            );
+
+        int key = 1;
+
+        List<ClusterNode> affNodes = (List<ClusterNode>) ig.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+        assert affNodes.size() == 2;
+
+        int primaryIdx = -1;
+
+        IgniteEx primary = null;
+        IgniteEx backup = null;
+
+        for (int i = 0; i < NODE_COUNT; i++) {
+            if (grid(i).localNode().equals(affNodes.get(0))) {
+                primaryIdx = i;
+                primary = grid(i);
+            }
+            else if (grid(i).localNode().equals(affNodes.get(1)))
+                backup = grid(i);
+        }
+
+        assert primary != null;
+        assert backup != null;
+
+        Integer val1 = 1;
+        Integer val2 = 2;
+
+        cache.put(key, val1);
+
+        assertEquals(val1, primary.cache(CACHE_NAME).get(key));
+        assertEquals(val1, backup.cache(CACHE_NAME).get(key));
+
+        if (ig == primary) {
+            ig = backup;
+
+            cache = ig.cache(CACHE_NAME);
+        }
+
+        primary.close();
+
+        assertEquals(backup.localNode(), ig.affinity(CACHE_NAME).mapKeyToNode(key));
+
+        cache.put(key, val2);
+
+        assertEquals(val2, backup.cache(CACHE_NAME).get(key));
+
+        primary = startGrid(primaryIdx);
+
+        assertEquals(backup.localNode(), ig.affinity(CACHE_NAME).mapKeyToNode(key));
+
+        primary.cache(CACHE_NAME).rebalance().get();
+
+        awaitPartitionMapExchange();
+
+        assertEquals(primary.localNode(), ig.affinity(CACHE_NAME).mapKeyToNode(key));
+
+        assertEquals(val2, primary.cache(CACHE_NAME).get(key));
+        assertEquals(val2, backup.cache(CACHE_NAME).get(key));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPrimaryLeftAndClusterRestart() throws Exception {
+        startGrids(NODE_COUNT);
+
+        IgniteEx ig = grid(0);
+
+        ig.active(true);
+
+        IgniteCache<Integer, Integer> cache =
+            ig.createCache(
+                new CacheConfiguration<Integer, Integer>()
+                    .setName(CACHE_NAME)
+                    .setWriteSynchronizationMode(FULL_SYNC)
+                    .setCacheMode(PARTITIONED)
+                    .setBackups(1)
+                    .setPartitionLossPolicy(READ_ONLY_SAFE)
+                    .setReadFromBackup(true)
+                    .setRebalanceDelay(-1)
+            );
+
+        int key = 1;
+
+        List<ClusterNode> affNodes = (List<ClusterNode>) ig.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+        assert affNodes.size() == 2;
+
+        int primaryIdx = -1;
+        int backupIdx = -1;
+
+        IgniteEx primary = null;
+        IgniteEx backup = null;
+
+        for (int i = 0; i < NODE_COUNT; i++) {
+            if (grid(i).localNode().equals(affNodes.get(0))) {
+                primaryIdx = i;
+                primary = grid(i);
+            }
+            else if (grid(i).localNode().equals(affNodes.get(1))) {
+                backupIdx = i;
+                backup = grid(i);
+            }
+        }
+
+        assert primary != null;
+        assert backup != null;
+
+        Integer val1 = 1;
+        Integer val2 = 2;
+
+        cache.put(key, val1);
+
+        assertEquals(val1, primary.cache(CACHE_NAME).get(key));
+        assertEquals(val1, backup.cache(CACHE_NAME).get(key));
+
+        if (ig == primary) {
+            ig = backup;
+
+            cache = ig.cache(CACHE_NAME);
+        }
+
+        stopGrid(primaryIdx, false);
+
+        assertEquals(backup.localNode(), ig.affinity(CACHE_NAME).mapKeyToNode(key));
+
+        cache.put(key, val2);
+
+        assertEquals(val2, backup.cache(CACHE_NAME).get(key));
+
+        stopAllGrids(false);
+
+        startGrids(NODE_COUNT);
+
+        ig = grid(0);
+        primary = grid(primaryIdx);
+        backup = grid(backupIdx);
+
+        boolean activated = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                for (int i = 0; i < NODE_COUNT; i++)
+                  if (!grid(i).active())
+                      return false;
+
+                return true;
+            }
+        }, 10_000);
+
+        assert activated;
+
+//        assertEquals(backup.localNode(), ig.affinity(CACHE_NAME).mapKeyToNode(key));
+
+        assertEquals(val2, primary.cache(CACHE_NAME).get(key));
+        assertEquals(val2, backup.cache(CACHE_NAME).get(key));
+
+        primary.cache(CACHE_NAME).rebalance().get();
+
+        affNodes = (List<ClusterNode>) ig.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+        assertEquals(primary.localNode(), affNodes.get(0));
+        assertEquals(backup.localNode(), affNodes.get(1));
+
+        assertEquals(val2, primary.cache(CACHE_NAME).get(key));
+        assertEquals(val2, backup.cache(CACHE_NAME).get(key));
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testMetadataUpdate() throws Exception {
+        startGrids(5);
+
+        Ignite ignite3 = grid(3);
+
+        ignite3.active(true);
+
+        CacheConfiguration<Object, Object> repCacheCfg = new CacheConfiguration<>("replicated")
+            .setCacheMode(CacheMode.REPLICATED)
+            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+        IgniteCache<Object, Object> cache = ignite3.getOrCreateCache(repCacheCfg);
+
+        stopGrid(0);
+        stopGrid(1);
+        stopGrid(2);
+        stopGrid(4);
+
+        for (int i = 0; i < 100; i++)
+            cache.put(i, new TestValue(i));
+
+        stopAllGrids();
+
+        startGrids(5);
+
+        GridTestUtils.waitForCondition(new PA() {
+            @Override
+            public boolean apply() {
+                return grid(0).cluster().active();
+            }
+        }, getTestTimeout());
+
+        for (int g = 0; g < 5; g++) {
+            for (int i = 0; i < 100; i++)
+                assertEquals(new TestValue(i), grid(g).cache("replicated").get(i));
+        }
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testClusterRestoredOnRestart() throws Exception {
+        startGrids(5);
+
+        Ignite ignite3 = grid(3);
+
+        ignite3.active(true);
+
+        stopGrid(0);
+
+        CacheConfiguration<Object, Object> cacheConfiguration = new CacheConfiguration<>("unknown_cache");
+
+        cacheConfiguration.setBackups(3);
+
+        IgniteCache<Object, Object> cache0 = ignite3.getOrCreateCache(cacheConfiguration);
+
+        for (int i = 0; i < 2048; i++)
+            cache0.put(i, 0);
+
+        awaitPartitionMapExchange();
+
+        stopAllGrids();
+
+        startGrids(5);
+
+        GridTestUtils.waitForCondition(new PA() {
+            @Override
+            public boolean apply() {
+                return grid(0).cluster().active();
+            }
+        }, getTestTimeout());
+
+        for (int g = 0; g < 5; g++) {
+            for (int i = 0; i < 2048; i++)
+                assertEquals("For key: " + i, 0, grid(g).cache("unknown_cache").get(i));
+        }
+    }
+
+    /** */
+    private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+        Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+        for (ClusterNode clN : clNodes)
+            res.add(clN);
+
+        return res;
+    }
+
+    /**
+     *
+     */
+    private static class TestValue {
+        /** */
+        int f1;
+
+        /** */
+        int f2;
+
+        /** */
+        int f3;
+
+        /** */
+        int f4;
+
+        /**
+         * @param a Init value.
+         */
+        private TestValue(int a) {
+            f1 = f2 = f3 = f4 = a;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+
+            if (!(o instanceof TestValue))
+                return false;
+
+            TestValue other = (TestValue)o;
+
+            return
+                f1 == other.f1 &&
+                f2 == other.f2 &&
+                f3 == other.f3 &&
+                f4 == other.f4;
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            int result = f1;
+
+            result = 31 * result + f2;
+            result = 31 * result + f3;
+            result = 31 * result + f4;
+
+            return result;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java
index f93d60c..9660a76 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java
@@ -82,6 +82,7 @@ import static org.apache.ignite.cache.CacheMode.PARTITIONED;
 import static org.apache.ignite.cache.CacheMode.REPLICATED;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
 import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
+import static org.apache.ignite.testframework.GridTestUtils.mergeExchangeWaitVersion;
 
 /**
  *
@@ -1178,15 +1179,6 @@ public class CacheExchangeMergeTest extends GridCommonAbstractTest {
     }
 
     /**
-     * @param node Node.
-     * @param topVer Ready exchange version to wait for before trying to merge exchanges.
-     */
-    private void mergeExchangeWaitVersion(Ignite node, long topVer) {
-        ((IgniteKernal)node).context().cache().context().exchange().mergeExchangesTestWaitVersion(
-            new AffinityTopologyVersion(topVer, 0));
-    }
-
-    /**
      * @throws Exception If failed.
      */
     private void checkCaches() throws Exception {

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
index ab07611..0704dbd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
@@ -53,7 +53,8 @@ import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.GridNodeOrderComparator;
+import org.apache.ignite.internal.cluster.NodeOrderComparator;
+import org.apache.ignite.internal.cluster.NodeOrderLegacyComparator;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.IgniteNodeAttributes;
@@ -2852,7 +2853,7 @@ public class CacheLateAffinityAssignmentTest extends GridCommonAbstractTest {
                     affNodes.add(n);
             }
 
-            Collections.sort(affNodes, GridNodeOrderComparator.INSTANCE);
+            Collections.sort(affNodes, NodeOrderComparator.getInstance());
 
             AffinityFunctionContext affCtx = new GridAffinityFunctionContextImpl(
                 affNodes,

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePutAllFailoverAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePutAllFailoverAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePutAllFailoverAbstractTest.java
index 9a0fb03..57a150f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePutAllFailoverAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePutAllFailoverAbstractTest.java
@@ -125,21 +125,7 @@ public abstract class CachePutAllFailoverAbstractTest extends GridCacheAbstractS
 
         final long endTime = System.currentTimeMillis() + TEST_TIME;
 
-        IgniteInternalFuture<Object> restartFut = GridTestUtils.runAsync(new Callable<Object>() {
-            @Override public Object call() throws Exception {
-                Thread.currentThread().setName("restart-thread");
-
-                while (!finished.get() && System.currentTimeMillis() < endTime) {
-                    startGrid(NODE_CNT);
-
-                    U.sleep(500);
-
-                    stopGrid(NODE_CNT);
-                }
-
-                return null;
-            }
-        });
+        IgniteInternalFuture<Object> restartFut = createAndRunConcurrentAction(finished, endTime);
 
         try {
             final IgniteCache<TestKey, TestValue> cache = ignite(0).cache(DEFAULT_CACHE_NAME);
@@ -235,6 +221,25 @@ public abstract class CachePutAllFailoverAbstractTest extends GridCacheAbstractS
         }
     }
 
+    /** */
+    protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean finished, final long endTime) {
+        return GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                Thread.currentThread().setName("restart-thread");
+
+                while (!finished.get() && System.currentTimeMillis() < endTime) {
+                    startGrid(NODE_CNT);
+
+                    U.sleep(500);
+
+                    stopGrid(NODE_CNT);
+                }
+
+                return null;
+            }
+        });
+    }
+
     /**
      *
      */

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
index 86c0fa4..34bd0a0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
@@ -32,6 +32,7 @@ import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.CI2;
+import org.apache.ignite.internal.util.typedef.PA;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -298,6 +299,12 @@ public class IgniteOptimisticTxSuspendResumeTest extends GridCommonAbstractTest
                         }
                     }).get(FUT_TIMEOUT);
 
+                    assertTrue(GridTestUtils.waitForCondition(new PA() {
+                        @Override public boolean apply() {
+                            return tx.state() == ROLLED_BACK;
+                        }
+                    }, getTestTimeout()));
+
                     assertEquals(ROLLED_BACK, tx.state());
 
                     assertFalse(cache.containsKey(1));
@@ -442,6 +449,12 @@ public class IgniteOptimisticTxSuspendResumeTest extends GridCommonAbstractTest
                         }
                     }, TransactionTimeoutException.class);
 
+                    assertTrue(GridTestUtils.waitForCondition(new PA() {
+                        @Override public boolean apply() {
+                            return tx.state() == ROLLED_BACK;
+                        }
+                    }, getTestTimeout()));
+
                     assertEquals(ROLLED_BACK, tx.state());
 
                     tx.close();
@@ -476,6 +489,12 @@ public class IgniteOptimisticTxSuspendResumeTest extends GridCommonAbstractTest
                         }
                     }, TransactionTimeoutException.class);
 
+                    assertTrue(GridTestUtils.waitForCondition(new PA() {
+                        @Override public boolean apply() {
+                            return tx.state() == ROLLED_BACK;
+                        }
+                    }, getTestTimeout()));
+
                     assertEquals(ROLLED_BACK, tx.state());
 
                     tx.close();


[11/17] ignite git commit: IEP-4 Baseline topology for persistent caches (Phase 1) Contributed by: Dmitriy Govorukhin Dmitry Pavlov Eduard Shangareev Ily

Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java
index d2f0099..bdca21c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java
@@ -19,16 +19,16 @@ package org.apache.ignite.internal.processors.cache.persistence.freelist;
 
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.Storable;
 
 /**
  */
-public interface FreeList {
+public interface FreeList<T extends Storable> {
     /**
      * @param row Row.
      * @throws IgniteCheckedException If failed.
      */
-    public void insertDataRow(CacheDataRow row) throws IgniteCheckedException;
+    public void insertDataRow(T row) throws IgniteCheckedException;
 
     /**
      * @param link Row link.
@@ -36,7 +36,7 @@ public interface FreeList {
      * @return {@code True} if was able to update row.
      * @throws IgniteCheckedException If failed.
      */
-    public boolean updateDataRow(long link, CacheDataRow row) throws IgniteCheckedException;
+    public boolean updateDataRow(long link, T row) throws IgniteCheckedException;
 
     /**
      * @param link Row link.

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java
deleted file mode 100644
index 5bacc53..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java
+++ /dev/null
@@ -1,619 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence.freelist;
-
-import java.util.concurrent.atomic.AtomicReferenceArray;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.internal.pagemem.PageIdAllocator;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertFragmentRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageRemoveRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageUpdateRecord;
-import org.apache.ignite.internal.processors.cache.CacheObject;
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
-import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
-import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
-import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictionTracker;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPagePayload;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseBag;
-import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
-import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
-import org.apache.ignite.internal.util.typedef.T2;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- */
-public class FreeListImpl extends PagesList implements FreeList, ReuseList {
-    /** */
-    private static final int BUCKETS = 256; // Must be power of 2.
-
-    /** */
-    private static final int REUSE_BUCKET = BUCKETS - 1;
-
-    /** */
-    private static final Integer COMPLETE = Integer.MAX_VALUE;
-
-    /** */
-    private static final Integer FAIL_I = Integer.MIN_VALUE;
-
-    /** */
-    private static final Long FAIL_L = Long.MAX_VALUE;
-
-    /** */
-    private static final int MIN_PAGE_FREE_SPACE = 8;
-
-    /** */
-    private final int shift;
-
-    /** */
-    private final AtomicReferenceArray<Stripe[]> buckets = new AtomicReferenceArray<>(BUCKETS);
-
-    /** */
-    private final int MIN_SIZE_FOR_DATA_PAGE;
-
-    /** */
-    private final int emptyDataPagesBucket;
-
-    /** */
-    private final PageHandler<CacheDataRow, Boolean> updateRow = new UpdateRowHandler();
-
-    /** */
-    private final DataRegionMetricsImpl memMetrics;
-
-    /** */
-    private final PageEvictionTracker evictionTracker;
-
-    /**
-     *
-     */
-    private final class UpdateRowHandler extends PageHandler<CacheDataRow, Boolean> {
-        @Override public Boolean run(
-            int cacheId,
-            long pageId,
-            long page,
-            long pageAddr,
-            PageIO iox,
-            Boolean walPlc,
-            CacheDataRow row,
-            int itemId)
-            throws IgniteCheckedException {
-            DataPageIO io = (DataPageIO)iox;
-
-            int rowSize = getRowSize(row, row.cacheId() != 0);
-
-            boolean updated = io.updateRow(pageAddr, itemId, pageSize(), null, row, rowSize);
-
-            evictionTracker.touchPage(pageId);
-
-            if (updated && needWalDeltaRecord(pageId, page, walPlc)) {
-                // TODO This record must contain only a reference to a logical WAL record with the actual data.
-                byte[] payload = new byte[rowSize];
-
-                DataPagePayload data = io.readPayload(pageAddr, itemId, pageSize());
-
-                assert data.payloadSize() == rowSize;
-
-                PageUtils.getBytes(pageAddr, data.offset(), payload, 0, rowSize);
-
-                wal.log(new DataPageUpdateRecord(
-                    cacheId,
-                    pageId,
-                    itemId,
-                    payload));
-            }
-
-            return updated;
-        }
-    }
-
-    /** */
-    private final PageHandler<CacheDataRow, Integer> writeRow = new WriteRowHandler();
-
-    /**
-     *
-     */
-    private final class WriteRowHandler extends PageHandler<CacheDataRow, Integer> {
-        @Override public Integer run(
-            int cacheId,
-            long pageId,
-            long page,
-            long pageAddr,
-            PageIO iox,
-            Boolean walPlc,
-            CacheDataRow row,
-            int written)
-            throws IgniteCheckedException {
-            DataPageIO io = (DataPageIO)iox;
-
-            int rowSize = getRowSize(row, row.cacheId() != 0);
-            int oldFreeSpace = io.getFreeSpace(pageAddr);
-
-            assert oldFreeSpace > 0 : oldFreeSpace;
-
-            // If the full row does not fit into this page write only a fragment.
-            written = (written == 0 && oldFreeSpace >= rowSize) ? addRow(pageId, page, pageAddr, io, row, rowSize):
-                addRowFragment(pageId, page, pageAddr, io, row, written, rowSize);
-
-            // Reread free space after update.
-            int newFreeSpace = io.getFreeSpace(pageAddr);
-
-            if (newFreeSpace > MIN_PAGE_FREE_SPACE) {
-                int bucket = bucket(newFreeSpace, false);
-
-                put(null, pageId, page, pageAddr, bucket);
-            }
-
-            if (written == rowSize)
-                evictionTracker.touchPage(pageId);
-
-            // Avoid boxing with garbage generation for usual case.
-            return written == rowSize ? COMPLETE : written;
-        }
-
-        /**
-         * @param pageId Page ID.
-         * @param page Page pointer.
-         * @param pageAddr Page address.
-         * @param io IO.
-         * @param row Row.
-         * @param rowSize Row size.
-         * @return Written size which is always equal to row size here.
-         * @throws IgniteCheckedException If failed.
-         */
-        private int addRow(
-            long pageId,
-            long page,
-            long pageAddr,
-            DataPageIO io,
-            CacheDataRow row,
-            int rowSize
-        ) throws IgniteCheckedException {
-            io.addRow(pageId, pageAddr, row, rowSize, pageSize());
-
-            if (needWalDeltaRecord(pageId, page, null)) {
-                // TODO IGNITE-5829 This record must contain only a reference to a logical WAL record with the actual data.
-                byte[] payload = new byte[rowSize];
-
-                DataPagePayload data = io.readPayload(pageAddr, PageIdUtils.itemId(row.link()), pageSize());
-
-                assert data.payloadSize() == rowSize;
-
-                PageUtils.getBytes(pageAddr, data.offset(), payload, 0, rowSize);
-
-                wal.log(new DataPageInsertRecord(
-                    grpId,
-                    pageId,
-                    payload));
-            }
-
-            return rowSize;
-        }
-
-        /**
-         * @param pageId Page ID.
-         * @param page Page pointer.
-         * @param pageAddr Page address.
-         * @param io IO.
-         * @param row Row.
-         * @param written Written size.
-         * @param rowSize Row size.
-         * @return Updated written size.
-         * @throws IgniteCheckedException If failed.
-         */
-        private int addRowFragment(
-            long pageId,
-            long page,
-            long pageAddr,
-            DataPageIO io,
-            CacheDataRow row,
-            int written,
-            int rowSize
-        ) throws IgniteCheckedException {
-            // Read last link before the fragment write, because it will be updated there.
-            long lastLink = row.link();
-
-            int payloadSize = io.addRowFragment(pageMem, pageAddr, row, written, rowSize, pageSize());
-
-            assert payloadSize > 0 : payloadSize;
-
-            if (needWalDeltaRecord(pageId, page, null)) {
-                // TODO IGNITE-5829 This record must contain only a reference to a logical WAL record with the actual data.
-                byte[] payload = new byte[payloadSize];
-
-                DataPagePayload data = io.readPayload(pageAddr, PageIdUtils.itemId(row.link()), pageSize());
-
-                PageUtils.getBytes(pageAddr, data.offset(), payload, 0, payloadSize);
-
-                wal.log(new DataPageInsertFragmentRecord(grpId, pageId, payload, lastLink));
-            }
-
-            return written + payloadSize;
-        }
-    }
-
-
-    /** */
-    private final PageHandler<Void, Long> rmvRow;
-
-    /**
-     *
-     */
-    private final class RemoveRowHandler extends PageHandler<Void, Long> {
-        /** Indicates whether partition ID should be masked from page ID. */
-        private final boolean maskPartId;
-
-        /** */
-        RemoveRowHandler(boolean maskPartId) {
-            this.maskPartId = maskPartId;
-        }
-
-        @Override public Long run(
-            int cacheId,
-            long pageId,
-            long page,
-            long pageAddr,
-            PageIO iox,
-            Boolean walPlc,
-            Void ignored,
-            int itemId)
-            throws IgniteCheckedException {
-            DataPageIO io = (DataPageIO)iox;
-
-            int oldFreeSpace = io.getFreeSpace(pageAddr);
-
-            assert oldFreeSpace >= 0: oldFreeSpace;
-
-            long nextLink = io.removeRow(pageAddr, itemId, pageSize());
-
-            if (needWalDeltaRecord(pageId, page, walPlc))
-                wal.log(new DataPageRemoveRecord(cacheId, pageId, itemId));
-
-            int newFreeSpace = io.getFreeSpace(pageAddr);
-
-            if (newFreeSpace > MIN_PAGE_FREE_SPACE) {
-                int newBucket = bucket(newFreeSpace, false);
-
-                if (oldFreeSpace > MIN_PAGE_FREE_SPACE) {
-                    int oldBucket = bucket(oldFreeSpace, false);
-
-                    if (oldBucket != newBucket) {
-                        // It is possible that page was concurrently taken for put, in this case put will handle bucket change.
-                        pageId = maskPartId ? PageIdUtils.maskPartitionId(pageId) : pageId;
-                        if (removeDataPage(pageId, page, pageAddr, io, oldBucket))
-                            put(null, pageId, page, pageAddr, newBucket);
-                    }
-                }
-                else
-                    put(null, pageId, page, pageAddr, newBucket);
-
-                if (io.isEmpty(pageAddr))
-                    evictionTracker.forgetPage(pageId);
-            }
-
-            // For common case boxed 0L will be cached inside of Long, so no garbage will be produced.
-            return nextLink;
-        }
-    }
-
-    /**
-     * @param cacheId Cache ID.
-     * @param name Name (for debug purpose).
-     * @param memMetrics Memory metrics.
-     * @param memPlc Data region.
-     * @param reuseList Reuse list or {@code null} if this free list will be a reuse list for itself.
-     * @param wal Write ahead log manager.
-     * @param metaPageId Metadata page ID.
-     * @param initNew {@code True} if new metadata should be initialized.
-     * @throws IgniteCheckedException If failed.
-     */
-    public FreeListImpl(
-        int cacheId,
-        String name,
-        DataRegionMetricsImpl memMetrics,
-        DataRegion memPlc,
-        ReuseList reuseList,
-        IgniteWriteAheadLogManager wal,
-        long metaPageId,
-        boolean initNew) throws IgniteCheckedException {
-        super(cacheId, name, memPlc.pageMemory(), BUCKETS, wal, metaPageId);
-
-        rmvRow = new RemoveRowHandler(cacheId == 0);
-
-        this.evictionTracker = memPlc.evictionTracker();
-        this.reuseList = reuseList == null ? this : reuseList;
-        int pageSize = pageMem.pageSize();
-
-        assert U.isPow2(pageSize) : "Page size must be a power of 2: " + pageSize;
-        assert U.isPow2(BUCKETS);
-        assert BUCKETS <= pageSize : pageSize;
-
-        // TODO this constant is used because currently we cannot reuse data pages as index pages
-        // TODO and vice-versa. It should be removed when data storage format is finalized.
-        MIN_SIZE_FOR_DATA_PAGE = pageSize - DataPageIO.MIN_DATA_PAGE_OVERHEAD;
-
-        int shift = 0;
-
-        while (pageSize > BUCKETS) {
-            shift++;
-            pageSize >>>= 1;
-        }
-
-        this.shift = shift;
-
-        this.memMetrics = memMetrics;
-
-        emptyDataPagesBucket = bucket(MIN_SIZE_FOR_DATA_PAGE, false);
-
-        init(metaPageId, initNew);
-    }
-
-    /**
-     * Calculates average fill factor over FreeListImpl instance.
-     *
-     * @return Tuple (numenator, denominator).
-     */
-    public T2<Long, Long> fillFactor() {
-        long pageSize = pageSize();
-
-        long totalSize = 0;
-        long loadSize = 0;
-
-        for (int b = BUCKETS - 2; b > 0; b--) {
-            long bsize = pageSize - ((REUSE_BUCKET - b) << shift);
-
-            long pages = bucketsSize[b].longValue();
-
-            loadSize += pages * (pageSize - bsize);
-
-            totalSize += pages * pageSize;
-        }
-
-        return totalSize == 0 ? new T2<>(0L, 0L) : new T2<>(loadSize, totalSize);
-    }
-
-    /** {@inheritDoc} */
-    @Override public void dumpStatistics(IgniteLogger log) {
-        long dataPages = 0;
-
-        final boolean dumpBucketsInfo = false;
-
-        for (int b = 0; b < BUCKETS; b++) {
-            long size = bucketsSize[b].longValue();
-
-            if (!isReuseBucket(b))
-                dataPages += size;
-
-            if (dumpBucketsInfo) {
-                Stripe[] stripes = getBucket(b);
-
-                boolean empty = true;
-
-                if (stripes != null) {
-                    for (Stripe stripe : stripes) {
-                        if (!stripe.empty) {
-                            empty = false;
-
-                            break;
-                        }
-                    }
-                }
-
-                if (log.isInfoEnabled())
-                    log.info("Bucket [b=" + b +
-                        ", size=" + size +
-                        ", stripes=" + (stripes != null ? stripes.length : 0) +
-                        ", stripesEmpty=" + empty + ']');
-            }
-        }
-
-        if (dataPages > 0) {
-            if (log.isInfoEnabled())
-                log.info("FreeList [name=" + name +
-                    ", buckets=" + BUCKETS +
-                    ", dataPages=" + dataPages +
-                    ", reusePages=" + bucketsSize[REUSE_BUCKET].longValue() + "]");
-        }
-    }
-
-    /**
-     * @param freeSpace Page free space.
-     * @param allowReuse {@code True} if it is allowed to get reuse bucket.
-     * @return Bucket.
-     */
-    private int bucket(int freeSpace, boolean allowReuse) {
-        assert freeSpace > 0 : freeSpace;
-
-        int bucket = freeSpace >>> shift;
-
-        assert bucket >= 0 && bucket < BUCKETS : bucket;
-
-        if (!allowReuse && isReuseBucket(bucket))
-            bucket--;
-
-        return bucket;
-    }
-
-    /**
-     * @param part Partition.
-     * @return Page ID.
-     * @throws IgniteCheckedException If failed.
-     */
-    private long allocateDataPage(int part) throws IgniteCheckedException {
-        assert part <= PageIdAllocator.MAX_PARTITION_ID;
-        assert part != PageIdAllocator.INDEX_PARTITION;
-
-        return pageMem.allocatePage(grpId, part, PageIdAllocator.FLAG_DATA);
-    }
-
-    /** {@inheritDoc} */
-    @Override public void insertDataRow(CacheDataRow row) throws IgniteCheckedException {
-        int rowSize = getRowSize(row, row.cacheId() != 0);
-
-        int written = 0;
-
-        do {
-            if (written != 0)
-                memMetrics.incrementLargeEntriesPages();
-
-            int freeSpace = Math.min(MIN_SIZE_FOR_DATA_PAGE, rowSize - written);
-
-            long pageId = 0L;
-
-            if (freeSpace == MIN_SIZE_FOR_DATA_PAGE)
-                pageId = takeEmptyPage(emptyDataPagesBucket, DataPageIO.VERSIONS);
-
-            boolean reuseBucket = false;
-
-            // TODO: properly handle reuse bucket.
-            if (pageId == 0L) {
-                for (int b = bucket(freeSpace, false) + 1; b < BUCKETS - 1; b++) {
-                    pageId = takeEmptyPage(b, DataPageIO.VERSIONS);
-
-                    if (pageId != 0L) {
-                        reuseBucket = isReuseBucket(b);
-
-                        break;
-                    }
-                }
-            }
-
-            boolean allocated = pageId == 0L;
-
-            if (allocated)
-                pageId = allocateDataPage(row.partition());
-            else
-                pageId = PageIdUtils.changePartitionId(pageId, (row.partition()));
-
-            DataPageIO init = reuseBucket || allocated ? DataPageIO.VERSIONS.latest() : null;
-
-            written = write(pageId, writeRow, init, row, written, FAIL_I);
-
-            assert written != FAIL_I; // We can't fail here.
-        }
-        while (written != COMPLETE);
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean updateDataRow(long link, CacheDataRow row) throws IgniteCheckedException {
-        assert link != 0;
-
-        long pageId = PageIdUtils.pageId(link);
-        int itemId = PageIdUtils.itemId(link);
-
-        Boolean updated = write(pageId, updateRow, row, itemId, null);
-
-        assert updated != null; // Can't fail here.
-
-        return updated;
-    }
-
-    /** {@inheritDoc} */
-    @Override public void removeDataRowByLink(long link) throws IgniteCheckedException {
-        assert link != 0;
-
-        long pageId = PageIdUtils.pageId(link);
-        int itemId = PageIdUtils.itemId(link);
-
-        long nextLink = write(pageId, rmvRow, itemId, FAIL_L);
-
-        assert nextLink != FAIL_L; // Can't fail here.
-
-        while (nextLink != 0L) {
-            memMetrics.decrementLargeEntriesPages();
-
-            itemId = PageIdUtils.itemId(nextLink);
-            pageId = PageIdUtils.pageId(nextLink);
-
-            nextLink = write(pageId, rmvRow, itemId, FAIL_L);
-
-            assert nextLink != FAIL_L; // Can't fail here.
-        }
-    }
-
-    /** {@inheritDoc} */
-    @Override protected Stripe[] getBucket(int bucket) {
-        return buckets.get(bucket);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected boolean casBucket(int bucket, Stripe[] exp, Stripe[] upd) {
-        return buckets.compareAndSet(bucket, exp, upd);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected boolean isReuseBucket(int bucket) {
-        return bucket == REUSE_BUCKET;
-    }
-
-    /**
-     * @return Number of empty data pages in free list.
-     */
-    public int emptyDataPages() {
-        return bucketsSize[emptyDataPagesBucket].intValue();
-    }
-
-    /** {@inheritDoc} */
-    @Override public void addForRecycle(ReuseBag bag) throws IgniteCheckedException {
-        assert reuseList == this: "not allowed to be a reuse list";
-
-        put(bag, 0, 0, 0L, REUSE_BUCKET);
-    }
-
-    /** {@inheritDoc} */
-    @Override public long takeRecycledPage() throws IgniteCheckedException {
-        assert reuseList == this: "not allowed to be a reuse list";
-
-        return takeEmptyPage(REUSE_BUCKET, null);
-    }
-
-    /** {@inheritDoc} */
-    @Override public long recycledPagesCount() throws IgniteCheckedException {
-        assert reuseList == this: "not allowed to be a reuse list";
-
-        return storedPagesCount(REUSE_BUCKET);
-    }
-
-    /**
-     * @param row Row.
-     * @param withCacheId If {@code true} adds cache ID size.
-     * @return Entry size on page.
-     * @throws IgniteCheckedException If failed.
-     */
-    public static int getRowSize(CacheDataRow row, boolean withCacheId) throws IgniteCheckedException {
-        KeyCacheObject key = row.key();
-        CacheObject val = row.value();
-
-        int keyLen = key.valueBytesLength(null);
-        int valLen = val.valueBytesLength(null);
-
-        return keyLen + valLen + CacheVersionIO.size(row.version(), false) + 8 + (withCacheId ? 4 : 0);
-    }
-
-    /** {@inheritDoc} */
-    @Override public String toString() {
-        return "FreeList [name=" + name + ']';
-    }
-}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
index b113c62..54ffb3e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
@@ -38,7 +38,7 @@ import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListSetPreviousR
 import org.apache.ignite.internal.processors.cache.persistence.DataStructure;
 import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListMetaIO;
 import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListNodeIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseBag;
@@ -240,11 +240,10 @@ public abstract class PagesList extends DataStructure {
                     }
 
                     boolean ok = casBucket(bucket, null, tails);
+
                     assert ok;
 
                     bucketsSize[bucket].set(bucketSize);
-
-                    changed = true;
                 }
             }
         }
@@ -695,7 +694,7 @@ public abstract class PagesList extends DataStructure {
             if (needWalDeltaRecord(pageId, page, null))
                 wal.log(new PagesListAddPageRecord(grpId, pageId, dataId));
 
-            DataPageIO dataIO = DataPageIO.VERSIONS.forPage(dataAddr);
+            AbstractDataPageIO dataIO = PageIO.getPageIO(dataAddr);
             dataIO.setFreeListPageId(dataAddr, pageId);
 
             if (needWalDeltaRecord(dataId, dataPage, null))
@@ -726,7 +725,7 @@ public abstract class PagesList extends DataStructure {
         final long dataAddr,
         int bucket
     ) throws IgniteCheckedException {
-        DataPageIO dataIO = DataPageIO.VERSIONS.forPage(dataAddr);
+        AbstractDataPageIO dataIO = PageIO.getPageIO(dataAddr);
 
         // Attempt to add page failed: the node page is full.
         if (isReuseBucket(bucket)) {
@@ -1149,7 +1148,7 @@ public abstract class PagesList extends DataStructure {
         final long dataId,
         final long dataPage,
         final long dataAddr,
-        DataPageIO dataIO,
+        AbstractDataPageIO dataIO,
         int bucket)
         throws IgniteCheckedException {
         final long pageId = dataIO.getFreeListPageId(dataAddr);

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
new file mode 100644
index 0000000..f0b2e16
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
@@ -0,0 +1,450 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord;
+import org.apache.ignite.internal.processors.cache.IncompleteObject;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.DbCheckpointListener;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.RootPage;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPagePayload;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.SimpleDataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.marshaller.Marshaller;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
+import org.jetbrains.annotations.NotNull;
+
+import static org.apache.ignite.internal.pagemem.PageIdUtils.itemId;
+import static org.apache.ignite.internal.pagemem.PageIdUtils.pageId;
+
+/**
+ * General purpose key-value local-only storage.
+ */
+public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, ReadWriteMetastorage {
+    /** */
+    public static final String METASTORAGE_CACHE_NAME = "MetaStorage";
+
+    /** */
+    public static final int METASTORAGE_CACHE_ID = CU.cacheId(METASTORAGE_CACHE_NAME);
+
+    /** */
+    private final IgniteWriteAheadLogManager wal;
+
+    /** */
+    private final DataRegion dataRegion;
+
+    /** */
+    private MetastorageTree tree;
+
+    /** */
+    private AtomicLong rmvId = new AtomicLong();
+
+    /** */
+    private DataRegionMetricsImpl regionMetrics;
+
+    /** */
+    private boolean readOnly;
+
+    /** */
+    private boolean empty;
+
+    /** */
+    private RootPage treeRoot;
+
+    /** */
+    private RootPage reuseListRoot;
+
+    /** */
+    private FreeListImpl freeList;
+
+    /** */
+    private Map<String, byte[]> lastUpdates;
+
+    /** */
+    private final Marshaller marshaller = new JdkMarshaller();
+
+    /** */
+    public MetaStorage(IgniteWriteAheadLogManager wal, DataRegion dataRegion, DataRegionMetricsImpl regionMetrics,
+        boolean readOnly) {
+        this.wal = wal;
+        this.dataRegion = dataRegion;
+        this.regionMetrics = regionMetrics;
+        this.readOnly = readOnly;
+    }
+
+    /** */
+    public MetaStorage(IgniteWriteAheadLogManager wal, DataRegion memPlc, DataRegionMetricsImpl memMetrics) {
+        this(wal, memPlc, memMetrics, false);
+    }
+
+    /** */
+    public void init(IgniteCacheDatabaseSharedManager db) throws IgniteCheckedException {
+        getOrAllocateMetas();
+
+        if (!empty) {
+            freeList = new FreeListImpl(METASTORAGE_CACHE_ID, "metastorage",
+                regionMetrics, dataRegion, null, wal, reuseListRoot.pageId().pageId(),
+                reuseListRoot.isAllocated());
+
+            MetastorageRowStore rowStore = new MetastorageRowStore(freeList, db);
+
+            tree = new MetastorageTree(METASTORAGE_CACHE_ID, dataRegion.pageMemory(), wal, rmvId,
+                freeList, rowStore, treeRoot.pageId().pageId(), treeRoot.isAllocated());
+
+            if (!readOnly)
+                ((GridCacheDatabaseSharedManager)db).addCheckpointListener(this);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public Serializable read(String key) throws IgniteCheckedException {
+        byte[] data = getData(key);
+
+        Object result = null;
+
+        if (data != null)
+            result = marshaller.unmarshal(data, getClass().getClassLoader());
+
+        return (Serializable) result;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void write(@NotNull String key, @NotNull Serializable val) throws IgniteCheckedException {
+        assert val != null;
+
+        byte[] data = marshaller.marshal(val);
+
+        putData(key, data);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void remove(@NotNull String key) throws IgniteCheckedException {
+        removeData(key);
+    }
+
+    /** */
+    public void putData(String key, byte[] data) throws IgniteCheckedException {
+        if (!readOnly) {
+            WALPointer ptr = wal.log(new MetastoreDataRecord(key, data));
+
+            wal.fsync(ptr);
+
+            synchronized (this) {
+                MetastorageDataRow oldRow = tree.findOne(new MetastorageDataRow(key, null));
+
+                if (oldRow != null) {
+                    tree.removex(oldRow);
+                    tree.rowStore().removeRow(oldRow.link());
+                }
+
+                MetastorageDataRow row = new MetastorageDataRow(key, data);
+                tree.rowStore().addRow(row);
+                tree.put(row);
+            }
+        }
+    }
+
+    /** */
+    public byte[] getData(String key) throws IgniteCheckedException {
+        if (readOnly) {
+            if (lastUpdates != null) {
+                byte[] lastUpdate = lastUpdates.get(key);
+
+                if (lastUpdate != null)
+                    return lastUpdate;
+            }
+
+            if (empty)
+                return null;
+        }
+
+        MetastorageDataRow row = tree.findOne(new MetastorageDataRow(key, null));
+
+        if (row == null)
+            return null;
+
+        return row.value();
+    }
+
+    /** */
+    public void removeData(String key) throws IgniteCheckedException {
+        if (!readOnly) {
+            WALPointer ptr = wal.log(new MetastoreDataRecord(key, null));
+
+            wal.fsync(ptr);
+
+            synchronized (this) {
+                MetastorageDataRow row = new MetastorageDataRow(key, null);
+                MetastorageDataRow oldRow = tree.findOne(row);
+
+                if (oldRow != null) {
+                    tree.removex(oldRow);
+                    tree.rowStore().removeRow(oldRow.link());
+                }
+            }
+        }
+    }
+
+    /** */
+    private void getOrAllocateMetas() throws IgniteCheckedException {
+        PageMemoryEx pageMem = (PageMemoryEx)dataRegion.pageMemory();
+
+        int partId = 0;
+
+        long partMetaId = pageMem.partitionMetaPageId(METASTORAGE_CACHE_ID, partId);
+        long partMetaPage = pageMem.acquirePage(METASTORAGE_CACHE_ID, partMetaId);
+        try {
+            if (readOnly) {
+                long pageAddr = pageMem.readLock(METASTORAGE_CACHE_ID, partMetaId, partMetaPage);
+
+                try {
+                    if (PageIO.getType(pageAddr) != PageIO.T_PART_META) {
+                        empty = true;
+
+                        return;
+                    }
+
+                    PagePartitionMetaIO io = PageIO.getPageIO(pageAddr);
+
+                    treeRoot = new RootPage(new FullPageId(io.getTreeRoot(pageAddr), METASTORAGE_CACHE_ID), false);
+                    reuseListRoot = new RootPage(new FullPageId(io.getReuseListRoot(pageAddr), METASTORAGE_CACHE_ID), false);
+                }
+                finally {
+                    pageMem.readUnlock(METASTORAGE_CACHE_ID, partId, partMetaPage);
+                }
+            }
+            else {
+                boolean allocated = false;
+                long pageAddr = pageMem.writeLock(METASTORAGE_CACHE_ID, partMetaId, partMetaPage);
+
+                try {
+                    long treeRoot, reuseListRoot;
+
+                    if (PageIO.getType(pageAddr) != PageIO.T_PART_META) {
+                        // Initialize new page.
+                        PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.latest();
+
+                        io.initNewPage(pageAddr, partMetaId, pageMem.pageSize());
+
+                        treeRoot = pageMem.allocatePage(METASTORAGE_CACHE_ID, partId, PageMemory.FLAG_DATA);
+                        reuseListRoot = pageMem.allocatePage(METASTORAGE_CACHE_ID, partId, PageMemory.FLAG_DATA);
+
+                        assert PageIdUtils.flag(treeRoot) == PageMemory.FLAG_DATA;
+                        assert PageIdUtils.flag(reuseListRoot) == PageMemory.FLAG_DATA;
+
+                        io.setTreeRoot(pageAddr, treeRoot);
+                        io.setReuseListRoot(pageAddr, reuseListRoot);
+
+                        if (PageHandler.isWalDeltaRecordNeeded(pageMem, METASTORAGE_CACHE_ID, partMetaId, partMetaPage, wal, null))
+                            wal.log(new MetaPageInitRecord(
+                                METASTORAGE_CACHE_ID,
+                                partMetaId,
+                                io.getType(),
+                                io.getVersion(),
+                                treeRoot,
+                                reuseListRoot
+                            ));
+
+                        allocated = true;
+                    }
+                    else {
+                        PagePartitionMetaIO io = PageIO.getPageIO(pageAddr);
+
+                        treeRoot = io.getTreeRoot(pageAddr);
+                        reuseListRoot = io.getReuseListRoot(pageAddr);
+
+                        assert PageIdUtils.flag(treeRoot) == PageMemory.FLAG_DATA :
+                            U.hexLong(treeRoot) + ", part=" + partId + ", METASTORAGE_CACHE_ID=" + METASTORAGE_CACHE_ID;
+                        assert PageIdUtils.flag(reuseListRoot) == PageMemory.FLAG_DATA :
+                            U.hexLong(reuseListRoot) + ", part=" + partId + ", METASTORAGE_CACHE_ID=" + METASTORAGE_CACHE_ID;
+                    }
+
+                    this.treeRoot = new RootPage(new FullPageId(treeRoot, METASTORAGE_CACHE_ID), allocated);
+                    this.reuseListRoot = new RootPage(new FullPageId(reuseListRoot, METASTORAGE_CACHE_ID), allocated);
+                }
+                finally {
+                    pageMem.writeUnlock(METASTORAGE_CACHE_ID, partMetaId, partMetaPage, null, allocated);
+                }
+            }
+        }
+        finally {
+            pageMem.releasePage(METASTORAGE_CACHE_ID, partMetaId, partMetaPage);
+        }
+    }
+
+    /**
+     * @return Page memory.
+     */
+    public PageMemory pageMemory() {
+        return dataRegion.pageMemory();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onCheckpointBegin(Context ctx) throws IgniteCheckedException {
+        freeList.saveMetadata();
+
+        MetastorageRowStore rowStore = tree.rowStore();
+
+        saveStoreMetadata(rowStore, ctx);
+    }
+
+    /**
+     * @param rowStore Store to save metadata.
+     * @throws IgniteCheckedException If failed.
+     */
+    private void saveStoreMetadata(MetastorageRowStore rowStore, Context ctx) throws IgniteCheckedException {
+        FreeListImpl freeList = (FreeListImpl)rowStore.freeList();
+
+        freeList.saveMetadata();
+    }
+
+    public void applyUpdate(String key, byte[] value) throws IgniteCheckedException {
+        if (readOnly) {
+            if (lastUpdates == null)
+                lastUpdates = new HashMap<>();
+
+            lastUpdates.put(key, value);
+        }
+        else {
+            if (value != null)
+                putData(key, value);
+            else
+                removeData(key);
+        }
+    }
+
+    /** */
+    public static class FreeListImpl extends AbstractFreeList<MetastorageDataRow> {
+        /** {@inheritDoc} */
+        FreeListImpl(int cacheId, String name, DataRegionMetricsImpl regionMetrics, DataRegion dataRegion,
+            ReuseList reuseList,
+            IgniteWriteAheadLogManager wal, long metaPageId, boolean initNew) throws IgniteCheckedException {
+            super(cacheId, name, regionMetrics, dataRegion, reuseList, wal, metaPageId, initNew);
+        }
+
+        /** {@inheritDoc} */
+        @Override public IOVersions<? extends AbstractDataPageIO<MetastorageDataRow>> ioVersions() {
+            return SimpleDataPageIO.VERSIONS;
+        }
+
+        /**
+         * Read row from data pages.
+         */
+        final MetastorageDataRow readRow(String key, long link)
+            throws IgniteCheckedException {
+            assert link != 0 : "link";
+
+            long nextLink = link;
+            IncompleteObject incomplete = null;
+            int size = 0;
+
+            boolean first = true;
+
+            do {
+                final long pageId = pageId(nextLink);
+
+                final long page = pageMem.acquirePage(grpId, pageId);
+
+                try {
+                    long pageAddr = pageMem.readLock(grpId, pageId, page); // Non-empty data page must not be recycled.
+
+                    assert pageAddr != 0L : nextLink;
+
+                    try {
+                        SimpleDataPageIO io = (SimpleDataPageIO)ioVersions().forPage(pageAddr);
+
+                        DataPagePayload data = io.readPayload(pageAddr, itemId(nextLink), pageMem.pageSize());
+
+                        nextLink = data.nextLink();
+
+                        if (first) {
+                            if (nextLink == 0) {
+                                // Fast path for a single page row.
+                                return new MetastorageDataRow(link, key, SimpleDataPageIO.readPayload(pageAddr + data.offset()));
+                            }
+
+                            first = false;
+                        }
+
+                        ByteBuffer buf = pageMem.pageBuffer(pageAddr);
+
+                        buf.position(data.offset());
+                        buf.limit(data.offset() + data.payloadSize());
+
+                        if (size == 0) {
+                            if (buf.remaining() >= 4 && incomplete == null) {
+                                // Just read size.
+                                size = buf.getInt();
+                                incomplete = new IncompleteObject(new byte[size]);
+                            }
+                            else {
+                                if (incomplete == null)
+                                    incomplete = new IncompleteObject(new byte[4]);
+
+                                incomplete.readData(buf);
+
+                                if (incomplete.isReady()) {
+                                    size = ByteBuffer.wrap(incomplete.data()).order(buf.order()).getInt();
+                                    incomplete = new IncompleteObject(new byte[size]);
+                                }
+                            }
+                        }
+
+                        if (size != 0 && buf.remaining() > 0)
+                            incomplete.readData(buf);
+                    }
+                    finally {
+                        pageMem.readUnlock(grpId, pageId, page);
+                    }
+                }
+                finally {
+                    pageMem.releasePage(grpId, pageId, page);
+                }
+            }
+            while (nextLink != 0);
+
+            assert incomplete.isReady();
+
+            return new MetastorageDataRow(link, key, incomplete.data());
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageDataRow.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageDataRow.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageDataRow.java
new file mode 100644
index 0000000..271efdf
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageDataRow.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+import org.apache.ignite.internal.processors.cache.persistence.Storable;
+
+/**
+ *
+ */
+public class MetastorageDataRow implements MetastorageSearchRow, Storable {
+    /** */
+    private long link;
+
+    /** */
+    private String key;
+
+    /** */
+    private byte[] value;
+
+    /** */
+    public MetastorageDataRow(long link, String key, byte[] value) {
+        this.link = link;
+        this.key = key;
+        this.value = value;
+    }
+
+    /** */
+    public MetastorageDataRow(String key, byte[] value) {
+        this.key = key;
+        this.value = value;
+    }
+
+    /**
+     * @return Key.
+     */
+    public String key() {
+        return key;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public int hash() {
+        return key.hashCode();
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public int partition() {
+        return 0;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public void link(long link) {
+        this.link = link;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public long link() {
+        return link;
+    }
+
+    /**
+     * @return Value.
+     */
+    public byte[] value() {
+        return value;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public String toString() {
+        return "key=" + key;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageLifecycleListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageLifecycleListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageLifecycleListener.java
new file mode 100644
index 0000000..8ab418c
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageLifecycleListener.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ * Listener for events of metastore lifecycle.
+ *
+ * Database manager is responsible for initializing metastore on node startup
+ * and notifying other components about its readiness.
+ */
+public interface MetastorageLifecycleListener {
+    /**
+     * Is called when metastorage is made ready for read-only operations very early on node startup phase.
+     *
+     * Reference for read-only metastorage should be used only within this method and shouldn't be stored
+     * to any field.
+     *
+     * @param metastorage Read-only meta storage.
+     */
+    public void onReadyForRead(ReadOnlyMetastorage metastorage) throws IgniteCheckedException;
+
+    /**
+     * Fully functional metastore capable of performing reading and writing operations.
+     *
+     * Components interested in using metastore are allowed to keep reference passed into the method
+     * in their fields.
+     *
+     * @param metastorage Fully functional meta storage.
+     */
+    public void onReadyForReadWrite(ReadWriteMetastorage metastorage) throws IgniteCheckedException;
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageRowStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageRowStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageRowStore.java
new file mode 100644
index 0000000..e4bafd5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageRowStore.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
+
+/**
+ *
+ */
+public class MetastorageRowStore {
+    /** */
+    private final FreeList freeList;
+
+    /** */
+    protected final IgniteCacheDatabaseSharedManager db;
+
+    /** */
+    public MetastorageRowStore(FreeList freeList, IgniteCacheDatabaseSharedManager db) {
+        this.freeList = freeList;
+        this.db = db;
+    }
+
+    /**
+     * @param link Row link.
+     * @return Data row.
+     */
+    public MetastorageDataRow dataRow(String key, long link) throws IgniteCheckedException {
+        return ((MetaStorage.FreeListImpl)freeList).readRow(key, link);
+    }
+
+    /**
+     * @param link Row link.
+     * @throws IgniteCheckedException If failed.
+     */
+    public void removeRow(long link) throws IgniteCheckedException {
+        assert link != 0;
+        db.checkpointReadLock();
+
+        try {
+            freeList.removeDataRowByLink(link);
+        }
+        finally {
+            db.checkpointReadUnlock();
+        }
+    }
+
+    /**
+     * @param row Row.
+     * @throws IgniteCheckedException If failed.
+     */
+    public void addRow(MetastorageDataRow row) throws IgniteCheckedException {
+        db.checkpointReadLock();
+
+        try {
+            freeList.insertDataRow(row);
+        }
+        finally {
+            db.checkpointReadUnlock();
+        }
+    }
+
+    /**
+     * @param link Row link.
+     * @param row New row data.
+     * @return {@code True} if was able to update row.
+     * @throws IgniteCheckedException If failed.
+     */
+    public boolean updateRow(long link, MetastorageDataRow row) throws IgniteCheckedException {
+        return freeList.updateDataRow(link, row);
+    }
+
+    /**
+     * @return Free list.
+     */
+    public FreeList freeList() {
+        return freeList;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageSearchRow.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageSearchRow.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageSearchRow.java
new file mode 100644
index 0000000..601fbc1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageSearchRow.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+/**
+ *
+ */
+public interface MetastorageSearchRow {
+    /**
+     * @return Key.
+     */
+    public String key();
+
+    /**
+     * @return Link for this row.
+     */
+    public long link();
+
+    /**
+     * @return Key hash code.
+     */
+    public int hash();
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java
new file mode 100644
index 0000000..445522b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+
+/**
+ *
+ */
+public class MetastorageTree extends BPlusTree<MetastorageSearchRow, MetastorageDataRow> {
+    /** Max key length (bytes num) */
+    public static final int MAX_KEY_LEN = 64;
+
+    /** */
+    private MetastorageRowStore rowStore;
+
+    /**
+     * @param pageMem
+     * @param wal
+     * @param globalRmvId
+     * @param metaPageId
+     * @param reuseList
+     * @throws IgniteCheckedException
+     */
+    public MetastorageTree(int cacheId,
+        PageMemory pageMem,
+        IgniteWriteAheadLogManager wal,
+        AtomicLong globalRmvId,
+        ReuseList reuseList,
+        MetastorageRowStore rowStore,
+        long metaPageId,
+        boolean initNew) throws IgniteCheckedException {
+        super("Metastorage", cacheId, pageMem, wal,
+            globalRmvId, metaPageId, reuseList, MetastorageInnerIO.VERSIONS, MetastoreLeafIO.VERSIONS);
+
+        this.rowStore = rowStore;
+
+        initTree(initNew);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected int compare(BPlusIO<MetastorageSearchRow> io, long pageAddr, int idx,
+        MetastorageSearchRow row) throws IgniteCheckedException {
+
+        String key = ((DataLinkIO)io).getKey(pageAddr, idx);
+
+        return key.compareTo(row.key());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected MetastorageDataRow getRow(BPlusIO<MetastorageSearchRow> io, long pageAddr, int idx,
+        Object x) throws IgniteCheckedException {
+        long link = ((DataLinkIO)io).getLink(pageAddr, idx);
+        String key = ((DataLinkIO)io).getKey(pageAddr, idx);
+
+        return rowStore.dataRow(key, link);
+    }
+
+    /**
+     * @return RowStore.
+     */
+    public MetastorageRowStore rowStore() {
+        return rowStore;
+    }
+
+    /**
+     *
+     */
+    private interface DataLinkIO {
+        /**
+         * @param pageAddr Page address.
+         * @param idx Index.
+         * @return Row link.
+         */
+        public long getLink(long pageAddr, int idx);
+
+        /**
+         * @param pageAddr Page address.
+         * @param idx Index.
+         * @return Key size in bytes.
+         */
+        public short getKeySize(long pageAddr, int idx);
+
+        /**
+         * @param pageAddr Page address.
+         * @param idx Index.
+         * @return Key.
+         */
+        public String getKey(long pageAddr, int idx);
+    }
+
+    /**
+     *
+     */
+    public static class MetastorageInnerIO extends BPlusInnerIO<MetastorageSearchRow> implements DataLinkIO {
+        /** */
+        public static final IOVersions<MetastorageInnerIO> VERSIONS = new IOVersions<>(
+            new MetastorageInnerIO(1)
+        );
+
+        /**
+         * @param ver Page format version.
+         */
+        MetastorageInnerIO(int ver) {
+            super(T_DATA_REF_METASTORAGE_INNER, ver, true, 10 + MAX_KEY_LEN);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void storeByOffset(long pageAddr, int off,
+            MetastorageSearchRow row) throws IgniteCheckedException {
+            assert row.link() != 0;
+
+            PageUtils.putLong(pageAddr, off, row.link());
+
+            byte[] bytes = row.key().getBytes();
+            assert bytes.length <= MAX_KEY_LEN;
+
+            PageUtils.putShort(pageAddr, off + 8, (short)bytes.length);
+            PageUtils.putBytes(pageAddr, off + 10, bytes);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void store(long dstPageAddr, int dstIdx, BPlusIO<MetastorageSearchRow> srcIo, long srcPageAddr,
+            int srcIdx) throws IgniteCheckedException {
+            int srcOff = srcIo.offset(srcIdx);
+            int dstOff = offset(dstIdx);
+
+            long link = ((DataLinkIO)srcIo).getLink(srcPageAddr, srcIdx);
+            short len = ((DataLinkIO)srcIo).getKeySize(srcPageAddr, srcIdx);
+
+            byte[] payload = PageUtils.getBytes(srcPageAddr, srcOff + 10, len);
+
+            PageUtils.putLong(dstPageAddr, dstOff, link);
+            PageUtils.putShort(dstPageAddr, dstOff + 8, len);
+            PageUtils.putBytes(dstPageAddr, dstOff + 10, payload);
+        }
+
+        /** {@inheritDoc} */
+        @Override public MetastorageSearchRow getLookupRow(BPlusTree<MetastorageSearchRow, ?> tree, long pageAddr,
+            int idx) throws IgniteCheckedException {
+            long link = getLink(pageAddr, idx);
+            String key = getKey(pageAddr, idx);
+
+            return new MetsatorageSearchRowImpl(key, link);
+        }
+
+        /** {@inheritDoc} */
+        @Override public long getLink(long pageAddr, int idx) {
+            assert idx < getCount(pageAddr) : idx;
+
+            return PageUtils.getLong(pageAddr, offset(idx));
+        }
+
+        /** {@inheritDoc} */
+        @Override public short getKeySize(long pageAddr, int idx) {
+            return PageUtils.getShort(pageAddr, offset(idx) + 8);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String getKey(long pageAddr, int idx) {
+            int len = PageUtils.getShort(pageAddr, offset(idx) + 8);
+            byte[] bytes = PageUtils.getBytes(pageAddr, offset(idx) + 10, len);
+            return new String(bytes);
+        }
+    }
+
+    /**
+     *
+     */
+    public static class MetastoreLeafIO extends BPlusLeafIO<MetastorageSearchRow> implements DataLinkIO {
+        /** */
+        public static final IOVersions<MetastoreLeafIO> VERSIONS = new IOVersions<>(
+            new MetastoreLeafIO(1)
+        );
+
+        /**
+         * @param ver Page format version.
+         */
+        MetastoreLeafIO(int ver) {
+            super(T_DATA_REF_METASTORAGE_LEAF, ver, 10 + MAX_KEY_LEN);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void storeByOffset(long pageAddr, int off,
+            MetastorageSearchRow row) throws IgniteCheckedException {
+            assert row.link() != 0;
+
+            PageUtils.putLong(pageAddr, off, row.link());
+            byte[] bytes = row.key().getBytes();
+
+            assert bytes.length <= MAX_KEY_LEN;
+
+            PageUtils.putShort(pageAddr, off + 8, (short)bytes.length);
+            PageUtils.putBytes(pageAddr, off + 10, bytes);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void store(long dstPageAddr, int dstIdx, BPlusIO<MetastorageSearchRow> srcIo, long srcPageAddr,
+            int srcIdx) throws IgniteCheckedException {
+            int srcOff = srcIo.offset(srcIdx);
+            int dstOff = offset(dstIdx);
+
+            long link = ((DataLinkIO)srcIo).getLink(srcPageAddr, srcIdx);
+            short len = ((DataLinkIO)srcIo).getKeySize(srcPageAddr, srcIdx);
+
+            byte[] payload = PageUtils.getBytes(srcPageAddr, srcOff + 10, len);
+
+            PageUtils.putLong(dstPageAddr, dstOff, link);
+            PageUtils.putShort(dstPageAddr, dstOff + 8, len);
+            PageUtils.putBytes(dstPageAddr, dstOff + 10, payload);
+        }
+
+        /** {@inheritDoc} */
+        @Override public MetastorageSearchRow getLookupRow(BPlusTree<MetastorageSearchRow, ?> tree, long pageAddr,
+            int idx) throws IgniteCheckedException {
+            long link = getLink(pageAddr, idx);
+            String key = getKey(pageAddr, idx);
+
+            return new MetsatorageSearchRowImpl(key, link);
+        }
+
+        /** {@inheritDoc} */
+        @Override public long getLink(long pageAddr, int idx) {
+            assert idx < getCount(pageAddr) : idx;
+
+            return PageUtils.getLong(pageAddr, offset(idx));
+        }
+
+        /** {@inheritDoc} */
+        @Override public short getKeySize(long pageAddr, int idx) {
+            return PageUtils.getShort(pageAddr, offset(idx) + 8);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String getKey(long pageAddr, int idx) {
+            int len = PageUtils.getShort(pageAddr, offset(idx) + 8);
+            byte[] bytes = PageUtils.getBytes(pageAddr, offset(idx) + 10, len);
+            return new String(bytes);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetsatorageSearchRowImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetsatorageSearchRowImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetsatorageSearchRowImpl.java
new file mode 100644
index 0000000..363c14d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetsatorageSearchRowImpl.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+/**
+ *
+ */
+public class MetsatorageSearchRowImpl implements MetastorageSearchRow {
+    /** */
+    private final String key;
+    /** */
+    private final long link;
+
+    /**
+     * @param key Key.
+     * @param link Link.
+     */
+    public MetsatorageSearchRowImpl(String key, long link) {
+        this.key = key;
+        this.link = link;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public String key() {
+        return key;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public long link() {
+        return link;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public int hash() {
+        return key.hashCode();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadOnlyMetastorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadOnlyMetastorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadOnlyMetastorage.java
new file mode 100644
index 0000000..9b73676
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadOnlyMetastorage.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+import java.io.Serializable;
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ *
+ */
+public interface ReadOnlyMetastorage {
+    /** */
+    Serializable read(String key) throws IgniteCheckedException;
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadWriteMetastorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadWriteMetastorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadWriteMetastorage.java
new file mode 100644
index 0000000..ab48afc
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadWriteMetastorage.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+import java.io.Serializable;
+import org.apache.ignite.IgniteCheckedException;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ *
+ */
+public interface ReadWriteMetastorage extends ReadOnlyMetastorage {
+    /** */
+    public void write(@NotNull String key, @NotNull Serializable val) throws IgniteCheckedException;
+
+    /** */
+    public void remove(@NotNull String key) throws IgniteCheckedException;
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
index 53e21b7..9c37508 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
@@ -33,17 +33,17 @@ import org.jetbrains.annotations.Nullable;
 public interface PageMemoryEx extends PageMemory {
     /**
      *
-     * @param cacheId Cache ID.
+     * @param grpId Group ID.
      * @param pageId Page ID.
      * @param page Page pointer.
      * @param restore Determines if the page is locked for restore.
      * @return ByteBuffer for modifying the page.
      */
-    long writeLock(int cacheId, long pageId, long page, boolean restore);
+    long writeLock(int grpId, long pageId, long page, boolean restore);
 
     /**
      *
-     * @param cacheId Cache ID.
+     * @param grpId Group ID.
      * @param pageId Page ID.
      * @param page Page pointer.
      * @param walPlc {@code True} if page should be recorded to WAL, {@code false} if the page must not
@@ -51,27 +51,27 @@ public interface PageMemoryEx extends PageMemory {
      * @param dirtyFlag Determines whether the page was modified since the last checkpoint.
      * @param restore Determines if the page is locked for restore.
      */
-    void writeUnlock(int cacheId, long pageId, long page, Boolean walPlc,
+    void writeUnlock(int grpId, long pageId, long page, Boolean walPlc,
         boolean dirtyFlag, boolean restore);
 
     /**
-     * Gets or allocates metadata page for specified cacheId.
+     * Gets or allocates metadata page for specified grpId.
      *
-     * @param cacheId Cache ID.
-     * @return Meta page for cacheId.
+     * @param grpId Group ID.
+     * @return Meta page for grpId.
      * @throws IgniteCheckedException If failed.
      */
-    public long metaPageId(int cacheId) throws IgniteCheckedException;
+    public long metaPageId(int grpId) throws IgniteCheckedException;
 
     /**
-     * Gets or allocates partition metadata page for specified cacheId and partId.
+     * Gets or allocates partition metadata page for specified grpId and partId.
      *
-     * @param cacheId Cache ID.
+     * @param grpId Group ID.
      * @param partId Partition ID.
-     * @return Meta page for cacheId and partId.
+     * @return Meta page for grpId and partId.
      * @throws IgniteCheckedException If failed.
      */
-    public long partitionMetaPageId(int cacheId, int partId) throws IgniteCheckedException;
+    public long partitionMetaPageId(int grpId, int partId) throws IgniteCheckedException;
 
     /**
      * @see #acquirePage(int, long)
@@ -125,11 +125,11 @@ public interface PageMemoryEx extends PageMemory {
     /**
      * Marks partition as invalid / outdated.
      *
-     * @param cacheId Cache ID.
+     * @param grpId Group ID.
      * @param partId Partition ID.
      * @return New partition tag (growing 1-based partition file version).
      */
-    public int invalidate(int cacheId, int partId);
+    public int invalidate(int grpId, int partId);
 
     /**
      * Clears internal metadata of destroyed cache group.

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 623bf54..90fde31 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.cache.persistence.pagemem;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -58,8 +59,8 @@ import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord;
 import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO;
@@ -77,6 +78,7 @@ import org.apache.ignite.internal.util.offheap.GridOffHeapOutOfMemoryException;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.jetbrains.annotations.Nullable;
 
 import static java.lang.Boolean.FALSE;
 import static java.lang.Boolean.TRUE;
@@ -123,7 +125,7 @@ public class PageMemoryImpl implements PageMemoryEx {
     /** Invalid relative pointer value. */
     private static final long INVALID_REL_PTR = RELATIVE_PTR_MASK;
 
-    /** */
+    /** Pointer which means that this page is outdated (for example, cache was destroyed, partition eviction'd happened */
     private static final long OUTDATED_REL_PTR = INVALID_REL_PTR + 1;
 
     /** Address mask to avoid ABA problem. */
@@ -214,8 +216,11 @@ public class PageMemoryImpl implements PageMemoryEx {
     /** Flush dirty page closure. When possible, will be called by evictPage(). */
     private final GridInClosure3X<FullPageId, ByteBuffer, Integer> flushDirtyPage;
 
-    /** Flush dirty page closure. When possible, will be called by evictPage(). */
-    private final GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker;
+    /**
+     * Flush dirty page closure. When possible, will be called by evictPage().
+     * {@code Null} if page tracking functionality is disabled
+     * */
+    @Nullable private final GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker;
 
     /** Pages write throttle. */
     private PagesWriteThrottle writeThrottle;
@@ -246,7 +251,7 @@ public class PageMemoryImpl implements PageMemoryEx {
         GridCacheSharedContext<?, ?> ctx,
         int pageSize,
         GridInClosure3X<FullPageId, ByteBuffer, Integer> flushDirtyPage,
-        GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker,
+        @Nullable GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker,
         CheckpointLockStateChecker stateChecker,
         DataRegionMetricsImpl memMetrics,
         boolean throttleEnabled
@@ -351,8 +356,10 @@ public class PageMemoryImpl implements PageMemoryEx {
 
         U.shutdownNow(getClass(), asyncRunner, log);
 
-        for (Segment seg : segments)
-            seg.close();
+        if (segments != null) {
+            for (Segment seg : segments)
+                seg.close();
+        }
 
         directMemoryProvider.shutdown();
     }
@@ -387,8 +394,8 @@ public class PageMemoryImpl implements PageMemoryEx {
     }
 
     /** {@inheritDoc} */
-    @Override public long writeLock(int cacheId, long pageId, long page, boolean restore) {
-        return writeLockPage(page, new FullPageId(pageId, cacheId), !restore);
+    @Override public long writeLock(int grpId, long pageId, long page, boolean restore) {
+        return writeLockPage(page, new FullPageId(pageId, grpId), !restore);
     }
 
     /** {@inheritDoc} */
@@ -403,9 +410,9 @@ public class PageMemoryImpl implements PageMemoryEx {
     }
 
     /** {@inheritDoc} */
-    @Override public void writeUnlock(int cacheId, long pageId, long page, Boolean walPlc,
+    @Override public void writeUnlock(int grpId, long pageId, long page, Boolean walPlc,
         boolean dirtyFlag, boolean restore) {
-        writeUnlockPage(page, new FullPageId(pageId, cacheId), walPlc, dirtyFlag, restore);
+        writeUnlockPage(page, new FullPageId(pageId, grpId), walPlc, dirtyFlag, restore);
     }
 
     /** {@inheritDoc} */
@@ -436,7 +443,7 @@ public class PageMemoryImpl implements PageMemoryEx {
 
         seg.writeLock().lock();
 
-        boolean isTrackingPage = trackingIO.trackingPageFor(pageId, pageSize()) == pageId;
+        boolean isTrackingPage = changeTracker != null && trackingIO.trackingPageFor(pageId, pageSize()) == pageId;
 
         try {
             long relPtr = seg.loadedPages.get(
@@ -555,12 +562,12 @@ public class PageMemoryImpl implements PageMemoryEx {
     }
 
     /** {@inheritDoc} */
-    @Override public long metaPageId(int cacheId) throws IgniteCheckedException {
-        return storeMgr.metaPageId(cacheId);
+    @Override public long metaPageId(int grpId) throws IgniteCheckedException {
+        return storeMgr.metaPageId(grpId);
     }
 
     /** {@inheritDoc} */
-    @Override public long partitionMetaPageId(int cacheId, int partId) throws IgniteCheckedException {
+    @Override public long partitionMetaPageId(int grpId, int partId) throws IgniteCheckedException {
         return PageIdUtils.pageId(partId, PageIdAllocator.FLAG_DATA, 0);
     }
 
@@ -846,9 +853,11 @@ public class PageMemoryImpl implements PageMemoryEx {
 
     /** {@inheritDoc} */
     @Override public boolean safeToUpdate() {
-        for (Segment segment : segments)
-            if (!segment.safeToUpdate())
-                return false;
+        if (segments != null) {
+            for (Segment segment : segments)
+                if (!segment.safeToUpdate())
+                    return false;
+        }
 
         return true;
     }
@@ -867,6 +876,9 @@ public class PageMemoryImpl implements PageMemoryEx {
 
     /** {@inheritDoc} */
     @Override public GridMultiCollectionWrapper<FullPageId> beginCheckpoint() throws IgniteException {
+        if (segments == null)
+            return new GridMultiCollectionWrapper<>(Collections.<FullPageId>emptyList());
+
         Collection[] collections = new Collection[segments.length];
 
         for (int i = 0; i < segments.length; i++) {
@@ -888,6 +900,9 @@ public class PageMemoryImpl implements PageMemoryEx {
     /** {@inheritDoc} */
     @SuppressWarnings({"unchecked", "TooBroadScope"})
     @Override public void finishCheckpoint() {
+        if (segments == null)
+            return;
+
         for (Segment seg : segments)
             seg.segCheckpointPages = null;
 
@@ -1073,14 +1088,14 @@ public class PageMemoryImpl implements PageMemoryEx {
     }
 
     /** {@inheritDoc} */
-    @Override public int invalidate(int cacheId, int partId) {
+    @Override public int invalidate(int grpId, int partId) {
         int tag = 0;
 
         for (Segment seg : segments) {
             seg.writeLock().lock();
 
             try {
-                int newTag = seg.incrementPartTag(cacheId, partId);
+                int newTag = seg.incrementPartTag(grpId, partId);
 
                 if (tag == 0)
                     tag = newTag;
@@ -1322,7 +1337,7 @@ public class PageMemoryImpl implements PageMemoryEx {
         boolean dirty = isDirty(page);
 
         //if page is for restore, we shouldn't mark it as changed
-        if (!restore && markDirty && !dirty)
+        if (!restore && markDirty && !dirty && changeTracker != null)
             changeTracker.apply(page, fullId, this);
 
         boolean pageWalRec = markDirty && walPlc != FALSE && (walPlc == TRUE || !dirty);

http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PagesAllocationRange.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PagesAllocationRange.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PagesAllocationRange.java
index e7170c3..f4508ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PagesAllocationRange.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PagesAllocationRange.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.partstate;
 
+import org.apache.ignite.internal.util.typedef.internal.S;
+
 /**
  * Range of pages allocated.
  * Contains currently allocated page count and previously observed page count.
@@ -60,9 +62,6 @@ public class PagesAllocationRange {
 
     /** {@inheritDoc} */
     @Override public String toString() {
-        return "PagesAllocationRange{" +
-            "lastAllocatedPageCnt=" + lastAllocatedPageCnt +
-            ", currAllocatedPageCnt=" + currAllocatedPageCnt +
-            '}';
+        return S.toString(PagesAllocationRange.class, this);
     }
 }