You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by as...@apache.org on 2020/05/19 07:27:12 UTC

[ignite] branch master updated: IGNITE-12938 Improve not idle cluster detection for IdleVerify and ValidateIndexes. - Fixes #7743.

This is an automated email from the ASF dual-hosted git repository.

ascherbakov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git


The following commit(s) were added to refs/heads/master by this push:
     new 832f741  IGNITE-12938 Improve not idle cluster detection for IdleVerify and ValidateIndexes. - Fixes #7743.
832f741 is described below

commit 832f7414e336b85b497e9bbce235dc2d3e21d78c
Author: zstan <st...@gmail.com>
AuthorDate: Tue May 19 10:22:24 2020 +0300

    IGNITE-12938 Improve not idle cluster detection for IdleVerify and ValidateIndexes. - Fixes #7743.
    
    Signed-off-by: Aleksei Scherbakov <as...@apache.org>
---
 .../processors/cache/PartitionUpdateCounter.java   |   5 +
 .../cache/PartitionUpdateCounterDebugWrapper.java  |   5 +
 .../cache/PartitionUpdateCounterTrackingImpl.java  |  13 ++
 .../cache/PartitionUpdateCounterVolatileImpl.java  |  10 +
 .../GridCacheDatabaseSharedManager.java            |  20 +-
 .../cache/persistence/pagemem/PageMemoryImpl.java  |   3 +-
 .../verify/CollectConflictPartitionKeysTask.java   |  21 +-
 .../cache/verify/IdleVerifyResultV2.java           |  10 +-
 .../processors/cache/verify/IdleVerifyUtility.java | 202 +++++++++++++++---
 .../cache/verify/PartitionHashRecordV2.java        |  12 +-
 .../cache/verify/VerifyBackupPartitionsTask.java   |  10 +-
 .../cache/verify/VerifyBackupPartitionsTaskV2.java | 165 ++++++--------
 .../ignite/internal/util/GridArgumentCheck.java    |   1 +
 .../verify/ValidateIndexesPartitionResult.java     |  92 +-------
 .../util/GridCommandHandlerAbstractTest.java       |  57 ++++-
 .../apache/ignite/util/GridCommandHandlerTest.java |  10 +-
 .../visor/verify/ValidateIndexesClosure.java       | 236 ++++++++++++---------
 .../processors/database/RebuildIndexTest.java      |   7 +-
 .../util/GridCommandHandlerIndexingTest.java       | 151 ++++++++++++-
 19 files changed, 671 insertions(+), 359 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounter.java
index d0f0502..7b8f1c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounter.java
@@ -143,4 +143,9 @@ public interface PartitionUpdateCounter extends Iterable<long[]> {
      * @return Cache group context.
      */
     public CacheGroupContext context();
+
+    /**
+     * @return A deep copy of current instance.
+     */
+    public PartitionUpdateCounter copy();
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterDebugWrapper.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterDebugWrapper.java
index 556e329..2c57c77 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterDebugWrapper.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterDebugWrapper.java
@@ -268,6 +268,11 @@ public class PartitionUpdateCounterDebugWrapper implements PartitionUpdateCounte
     }
 
     /** {@inheritDoc} */
+    @Override public PartitionUpdateCounter copy() {
+        return new PartitionUpdateCounterDebugWrapper(partId, delegate.copy());
+    }
+
+    /** {@inheritDoc} */
     @Override public String toString() {
         return delegate.toString();
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterTrackingImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterTrackingImpl.java
index 9a5f1b2..91927fb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterTrackingImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterTrackingImpl.java
@@ -454,4 +454,17 @@ public class PartitionUpdateCounterTrackingImpl implements PartitionUpdateCounte
     @Override public CacheGroupContext context() {
         return grp;
     }
+
+    /** {@inheritDoc} */
+    @Override public PartitionUpdateCounter copy() {
+        PartitionUpdateCounterTrackingImpl copy = new PartitionUpdateCounterTrackingImpl(grp);
+
+        copy.cntr.set(cntr.get());
+        copy.first = first;
+        copy.queue = new TreeMap<>(queue);
+        copy.initCntr = initCntr;
+        copy.reserveCntr.set(reserveCntr.get());
+
+        return copy;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterVolatileImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterVolatileImpl.java
index f25f456..ae32a51 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterVolatileImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterVolatileImpl.java
@@ -167,4 +167,14 @@ public class PartitionUpdateCounterVolatileImpl implements PartitionUpdateCounte
     @Override public CacheGroupContext context() {
         return grp;
     }
+
+    /** {@inheritDoc} */
+    @Override public PartitionUpdateCounter copy() {
+        PartitionUpdateCounterVolatileImpl copy = new PartitionUpdateCounterVolatileImpl(grp);
+
+        copy.cntr.set(cntr.get());
+        copy.initCntr = this.initCntr;
+
+        return copy;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 7e2a0ae..792b62d 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -542,8 +542,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
 
         int pagesNum = 0;
 
-        boolean hasUserDirtyPages = false;
-
         for (DataRegion reg : dataRegions()) {
             if (!reg.config().isPersistenceEnabled())
                 continue;
@@ -560,7 +558,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
         if (progress != null)
             progress.currentCheckpointPagesCount(pagesNum);
 
-        return new CheckpointPagesInfoHolder(res, pagesNum, hasUserDirtyPages);
+        return new CheckpointPagesInfoHolder(res, pagesNum);
     }
 
     /**
@@ -4010,8 +4008,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
 
                 dirtyPagesCount = cpPagesHolder.pagesNum();
 
-                hasUserPages = !cpPagesHolder.onlySystemPages();
-
                 hasPartitionsToDestroy = !curr.getDestroyQueue().pendingReqs().isEmpty();
 
                 WALPointer cpPtr = null;
@@ -4042,7 +4038,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
                 tracker.onLockRelease();
             }
 
-            DbCheckpointListener.Context ctx = createOnCheckpointBeginContext(ctx0, hasUserPages);
+            DbCheckpointListener.Context ctx = createOnCheckpointBeginContext(ctx0, dirtyPagesCount > 0);
 
             curr.transitTo(LOCK_RELEASED);
 
@@ -5582,9 +5578,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
 
     /** Current checkpoint pages information. */
     private static class CheckpointPagesInfoHolder {
-        /** If {@code true} there are user pages in checkpoint. */
-        private final boolean hasUserDirtyPages;
-
         /** Total pages count in cp. */
         private final int pagesNum;
 
@@ -5594,16 +5587,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
         /** */
         private CheckpointPagesInfoHolder(
             Collection<Map.Entry<PageMemoryEx, GridMultiCollectionWrapper<FullPageId>>> pages,
-            int num,
-            boolean hasUserPages) {
+            int num) {
             cpPages = pages;
             pagesNum = num;
-            hasUserDirtyPages = hasUserPages;
-        }
-
-        /** If {@code true} there are user pages in checkpoint. */
-        private boolean onlySystemPages() {
-            return !hasUserDirtyPages;
         }
 
         /** Total pages count in cp. */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index ff17a36..b17989cd 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -1119,7 +1119,7 @@ public class PageMemoryImpl implements PageMemoryEx {
         IgniteInternalFuture allowToReplace
     ) throws IgniteException {
         if (segments == null)
-            return new GridMultiCollectionWrapper<>(Collections.<FullPageId>emptyList());
+            return new GridMultiCollectionWrapper<>(Collections.emptyList());
 
         Collection[] collections = new Collection[segments.length];
 
@@ -1669,7 +1669,6 @@ public class PageMemoryImpl implements PageMemoryEx {
     /**
      * @param page Page pointer.
      * @param fullId full page ID.
-     * @param walPlc WAL policy
      * @param walPlc Full page WAL record policy
      * @param markDirty set dirty flag to page
      * @param restore restore flag
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java
index 8e8128c..25d12c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java
@@ -37,6 +37,7 @@ import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.CacheObjectUtils;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.PartitionUpdateCounter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
@@ -48,6 +49,8 @@ import org.apache.ignite.resources.LoggerResource;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.GRID_NOT_IDLE_MSG;
+
 /**
  *
  */
@@ -167,14 +170,16 @@ public class CollectConflictPartitionKeysTask extends ComputeTaskAdapter<Partiti
 
             int partHash = 0;
             long partSize;
-            long updateCntrBefore;
+            PartitionUpdateCounter updateCntrBefore;
             List<PartitionEntryHashRecord> partEntryHashRecords;
 
             try {
                 if (part.state() != GridDhtPartitionState.OWNING)
                     return Collections.emptyMap();
 
-                updateCntrBefore = part.updateCounter();
+                @Nullable PartitionUpdateCounter updCntr = part.dataStore().partUpdateCounter();
+
+                updateCntrBefore = updCntr == null ? null : updCntr.copy();
 
                 partSize = part.dataStore().fullSize();
 
@@ -202,12 +207,12 @@ public class CollectConflictPartitionKeysTask extends ComputeTaskAdapter<Partiti
                         row.key().valueBytes(grpCtx.cacheObjectContext()), row.version(), valHash));
                 }
 
-                long updateCntrAfter = part.updateCounter();
+                PartitionUpdateCounter updateCntrAfter = part.dataStore().partUpdateCounter();
 
-                if (updateCntrBefore != updateCntrAfter) {
-                    throw new IgniteException("Cluster is not idle: update counter of partition " + partKey.toString() +
-                        " changed during hash calculation [before=" + updateCntrBefore +
-                        ", after=" + updateCntrAfter + "]");
+                if (updateCntrAfter != null && !updateCntrAfter.equals(updateCntrBefore)) {
+                    throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[grpName=" + grpCtx.cacheOrGroupName() +
+                        ", grpId=" + grpCtx.groupId() + ", partId=" + part.id() + "] changed during hash calculation " +
+                        "[before=" + updateCntrBefore + ", after=" + updateCntrAfter + "]");
                 }
             }
             catch (IgniteCheckedException e) {
@@ -224,7 +229,7 @@ public class CollectConflictPartitionKeysTask extends ComputeTaskAdapter<Partiti
             boolean isPrimary = part.primary(grpCtx.topology().readyTopologyVersion());
 
             PartitionHashRecord partHashRec = new PartitionHashRecord(
-                partKey, isPrimary, consId, partHash, updateCntrBefore, partSize);
+                partKey, isPrimary, consId, partHash, updateCntrBefore == null ? 0 : updateCntrBefore.get(), partSize);
 
             Map<PartitionHashRecord, List<PartitionEntryHashRecord>> res = new HashMap<>();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
index ece5b22..5d2a5d3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
@@ -24,6 +24,7 @@ import java.io.ObjectOutput;
 import java.io.PrintWriter;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Consumer;
@@ -143,7 +144,7 @@ public class IdleVerifyResultV2 extends VisorDataTransferObject {
      * @return Moving partitions.
      */
     public Map<PartitionKeyV2, List<PartitionHashRecordV2>> movingPartitions() {
-        return movingPartitions;
+        return Collections.unmodifiableMap(movingPartitions);
     }
 
     /**
@@ -236,7 +237,12 @@ public class IdleVerifyResultV2 extends VisorDataTransferObject {
             else
                 printConflicts(printer);
 
-            printSkippedPartitions(printer, movingPartitions(), "MOVING");
+            Map<PartitionKeyV2, List<PartitionHashRecordV2>> moving = movingPartitions();
+
+            if (!moving.isEmpty())
+                printer.accept("Possible results are not full due to rebalance still in progress." + U.nl());
+
+            printSkippedPartitions(printer, moving, "MOVING");
             printSkippedPartitions(printer, lostPartitions(), "LOST");
         }
         else {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java
index 10835f9..b93c122 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java
@@ -19,16 +19,28 @@ package org.apache.ignite.internal.processors.cache.verify;
 
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
-import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointProgress;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.PartitionUpdateCounter;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.SB;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -36,25 +48,24 @@ import org.jetbrains.annotations.Nullable;
  */
 public class IdleVerifyUtility {
     /** Cluster not idle message. */
-    public static final String CLUSTER_NOT_IDLE_MSG = "Checkpoint with dirty pages started! Cluster not idle!";
+    public static final String GRID_NOT_IDLE_MSG =
+        "Cluster not idle. Modifications found in caches or groups: ";
 
     /**
-     * See {@link IdleVerifyUtility#checkPartitionsPageCrcSum(FilePageStore, CacheGroupContext, int, byte,
-     * AtomicBoolean)}.
+     * See {@link IdleVerifyUtility#checkPartitionsPageCrcSum(FilePageStore, CacheGroupContext, int, byte)}.
      */
     public static void checkPartitionsPageCrcSum(
         @Nullable FilePageStoreManager pageStoreMgr,
         CacheGroupContext grpCtx,
         int partId,
-        byte pageType,
-        AtomicBoolean cpFlag
+        byte pageType
     ) throws IgniteCheckedException, GridNotIdleException {
         if (!grpCtx.persistenceEnabled() || pageStoreMgr == null)
             return;
 
         FilePageStore pageStore = (FilePageStore)pageStoreMgr.getStore(grpCtx.groupId(), partId);
 
-        checkPartitionsPageCrcSum(pageStore, grpCtx, partId, pageType, cpFlag);
+        checkPartitionsPageCrcSum(pageStore, grpCtx, partId, pageType);
     }
 
     /**
@@ -65,7 +76,6 @@ public class IdleVerifyUtility {
      * @param grpCtx Passed cache group context.
      * @param partId Partition id.
      * @param pageType Page type. Possible types {@link PageIdAllocator#FLAG_DATA}, {@link PageIdAllocator#FLAG_IDX}.
-     * @param cpFlag Checkpoint flag for detecting start checkpoint with dirty pages.
      * @throws IgniteCheckedException If reading page failed.
      * @throws GridNotIdleException If cluster not idle.
      */
@@ -73,8 +83,7 @@ public class IdleVerifyUtility {
         FilePageStore pageStore,
         CacheGroupContext grpCtx,
         int partId,
-        byte pageType,
-        AtomicBoolean cpFlag
+        byte pageType
     ) throws IgniteCheckedException, GridNotIdleException {
         assert pageType == PageIdAllocator.FLAG_DATA || pageType == PageIdAllocator.FLAG_IDX : pageType;
 
@@ -87,28 +96,169 @@ public class IdleVerifyUtility {
         for (int pageNo = 0; pageNo < pageStore.pages(); pageId++, pageNo++) {
             buf.clear();
 
-            if (cpFlag.get())
-                throw new GridNotIdleException(CLUSTER_NOT_IDLE_MSG);
-
             pageStore.read(pageId, buf, true);
         }
+    }
+
+    /**
+     * Gather updateCounters info.
+     * Holds {@link org.apache.ignite.internal.processors.cache.PartitionUpdateCounter#copy} of update counters.
+     *
+     * @param ign Ignite instance.
+     * @param grpIds Group Id`s.
+     * @return Current groups distribution with update counters per partitions.
+     */
+    public static Map<Integer, Map<Integer, PartitionUpdateCounter>> getUpdateCountersSnapshot(
+        IgniteEx ign,
+        Set<Integer> grpIds
+    ) {
+        Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCountersPerGrp = new HashMap<>();
+
+        for (Integer grpId : grpIds) {
+            CacheGroupContext grpCtx = ign.context().cache().cacheGroup(grpId);
+
+            if (grpCtx == null)
+                throw new GridNotIdleException("Group not found: " + grpId + "."
+                        + " Possible reasons: rebalance in progress or concurrent cache destroy.");
+
+            GridDhtPartitionTopology top = grpCtx.topology();
+
+            Map<Integer, PartitionUpdateCounter> partsWithCounters =
+                partsWithCountersPerGrp.computeIfAbsent(grpId, k -> new HashMap<>());
+
+            for (GridDhtLocalPartition part : top.currentLocalPartitions()) {
+                if (part.state() != GridDhtPartitionState.OWNING)
+                    continue;
+
+                @Nullable PartitionUpdateCounter updCntr = part.dataStore().partUpdateCounter();
+
+                partsWithCounters.put(part.id(), updCntr == null ? null : updCntr.copy());
+            }
+        }
+
+        return partsWithCountersPerGrp;
+    }
+
+    /**
+     * Prints diff between incoming update counters snapshots.
+     *
+     * @param ig Ignite instance.
+     * @param diff Compared groups diff.
+     * @return Formatted diff representation.
+     */
+    public static String formatUpdateCountersDiff(IgniteEx ig, List<Integer> diff) {
+        SB sb = null;
+
+        if (!diff.isEmpty()) {
+            sb = new SB();
+
+            for (int grpId0 : diff) {
+                if (sb.length() != 0)
+                    sb.a(", ");
+                else
+                    sb.a("\"");
+
+                DynamicCacheDescriptor desc = ig.context().cache().cacheDescriptor(grpId0);
+
+                CacheGroupContext grpCtx = ig.context().cache().cacheGroup(desc == null ? grpId0 : desc.groupId());
+
+                sb.a(grpCtx.cacheOrGroupName());
+            }
+
+            sb.a("\"");
+        }
+
+        return sb != null ? sb.toString() : "";
+    }
+
+    /**
+     * Compares two sets with partitions and upd counters per group distribution.
+     *
+     * @param ign Ignite instance.
+     * @param cntrsIn Group id`s with counters per partitions per groups distribution.
+     * @param grpId Group id to compare.
+     * @return Diff with grpId info between two sets.
+     */
+    public static List<Integer> compareUpdateCounters(
+        IgniteEx ign,
+        Map<Integer, Map<Integer, PartitionUpdateCounter>> cntrsIn,
+        Integer grpId
+    ) {
+        Map<Integer, Map<Integer, PartitionUpdateCounter>> curCntrs =
+            getUpdateCountersSnapshot(ign, Collections.singleton(grpId));
 
-        if (cpFlag.get())
-            throw new GridNotIdleException(CLUSTER_NOT_IDLE_MSG);
+        if (curCntrs.isEmpty())
+            throw new GridNotIdleException("No OWNING partitions for group: " + grpId);
+
+        return compareUpdateCounters(ign, cntrsIn, curCntrs);
     }
 
     /**
-     * @param db Shared DB manager.
-     * @return {@code True} if checkpoint is now, {@code False} otherwise.
+     * Compares two sets with partitions and upd counters per group distribution.
+     *
+     * @param ign Ignite instance.
+     * @param cntrsEth Ethalon group id`s with counters per partitions per groups distribution.
+     * @param curCntrs Group id`s with counters per partitions per groups distribution compare with.
+     * @return Diff with grpId info between two sets.
      */
-    public static boolean isCheckpointNow(@Nullable IgniteCacheDatabaseSharedManager db) {
-        if (!(db instanceof GridCacheDatabaseSharedManager))
-            return false;
+    public static List<Integer> compareUpdateCounters(
+        IgniteEx ign,
+        Map<Integer, Map<Integer, PartitionUpdateCounter>> cntrsEth,
+        Map<Integer, Map<Integer, PartitionUpdateCounter>> curCntrs
+    ) {
+        List<Integer> diff = new ArrayList<>();
+
+        Integer grpId;
+        for (Map.Entry<Integer, Map<Integer, PartitionUpdateCounter>> curEntry : curCntrs.entrySet()) {
+            Map<Integer, PartitionUpdateCounter> partsWithCntrsCur = curEntry.getValue();
+
+            grpId = curEntry.getKey();
 
-        CheckpointProgress progress =
-            ((GridCacheDatabaseSharedManager)db).getCheckpointer().currentProgress();
+            if (partsWithCntrsCur == null)
+                throw new GridNotIdleException("Group not found: " + grpId + "."
+                    + " Possible reasons: rebalance in progress or concurrent cache destroy.");
+
+            Map<Integer, PartitionUpdateCounter> partsWithCntrsIn = cntrsEth.get(grpId);
+
+            if (!partsWithCntrsIn.equals(partsWithCntrsCur))
+                diff.add(grpId);
+        }
 
-        return progress != null && progress.inProgress();
+        return diff;
+    }
+
+    /**
+     * Idle checker.
+     */
+    public static class IdleChecker implements IgniteInClosure<Integer> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private final IgniteEx ig;
+
+        /** Group id`s snapshot with partitions and counters distrubution. */
+        private final Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrp;
+
+        /** */
+        public IdleChecker(IgniteEx ig, Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrp) {
+            this.ig = ig;
+            this.partsWithCntrsPerGrp = partsWithCntrsPerGrp;
+        }
+
+        /** */
+        @Override public void apply(Integer grpId) {
+            List<Integer> diff;
+
+            diff = compareUpdateCounters(ig, partsWithCntrsPerGrp, grpId);
+
+            if (!F.isEmpty(diff)) {
+                String res = formatUpdateCountersDiff(ig, diff);
+
+                if (!res.isEmpty())
+                    throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[" + res + "]");
+            }
+        }
     }
 
     /** */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecordV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecordV2.java
index 5cefa66..ce1d1c3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecordV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecordV2.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.cache.verify;
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -70,8 +71,15 @@ public class PartitionHashRecordV2 extends VisorDataTransferObject {
      * @param size Size.
      * @param partitionState Partition state.
      */
-    public PartitionHashRecordV2(PartitionKeyV2 partKey, boolean isPrimary,
-        Object consistentId, int partHash, long updateCntr, long size, PartitionState partitionState) {
+    public PartitionHashRecordV2(
+        PartitionKeyV2 partKey,
+        boolean isPrimary,
+        Object consistentId,
+        int partHash,
+        long updateCntr,
+        long size,
+        PartitionState partitionState
+    ) {
         this.partKey = partKey;
         this.isPrimary = isPrimary;
         this.consistentId = consistentId;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
index f700567..67b1146 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
@@ -56,6 +56,8 @@ import org.apache.ignite.resources.LoggerResource;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.GRID_NOT_IDLE_MSG;
+
 /**
  * Task for comparing update counters and checksums between primary and backup partitions of specified caches.
  * <br>
@@ -103,7 +105,7 @@ public class VerifyBackupPartitionsTask extends ComputeTaskAdapter<Set<String>,
 
             for (Map.Entry<PartitionKey, PartitionHashRecord> e : nodeHashes.entrySet()) {
                 if (!clusterHashes.containsKey(e.getKey()))
-                    clusterHashes.put(e.getKey(), new ArrayList<PartitionHashRecord>());
+                    clusterHashes.put(e.getKey(), new ArrayList<>());
 
                 clusterHashes.get(e.getKey()).add(e.getValue());
             }
@@ -330,9 +332,9 @@ public class VerifyBackupPartitionsTask extends ComputeTaskAdapter<Set<String>,
                 long updateCntrAfter = part.updateCounter();
 
                 if (updateCntrBefore != updateCntrAfter) {
-                    throw new IgniteException("Cluster is not idle: update counter of partition [grpId=" +
-                        grpCtx.groupId() + ", partId=" + part.id() + "] changed during hash calculation [before=" +
-                        updateCntrBefore + ", after=" + updateCntrAfter + "]");
+                    throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[grpName=" + grpCtx.cacheOrGroupName() +
+                        ", grpId=" + grpCtx.groupId() + ", partId=" + part.id() + "] changed during hash calculation " +
+                        "[before=" + updateCntrBefore + ", after=" + updateCntrAfter + "]");
                 }
             }
             catch (IgniteCheckedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
index 187dc8ce..ed405a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
@@ -30,7 +30,6 @@ import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Pattern;
 import org.apache.ignite.IgniteCheckedException;
@@ -50,11 +49,10 @@ import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheUtils;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.PartitionUpdateCounter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
-import org.apache.ignite.internal.processors.cache.persistence.DbCheckpointListener;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecordV2.PartitionState;
@@ -74,6 +72,7 @@ import org.jetbrains.annotations.Nullable;
 import static java.util.Collections.emptyMap;
 import static org.apache.ignite.cache.CacheMode.LOCAL;
 import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA;
+import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.GRID_NOT_IDLE_MSG;
 
 /**
  * Task for comparing update counters and checksums between primary and backup partitions of specified caches.
@@ -252,97 +251,68 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
 
         /** {@inheritDoc} */
         @Override public Map<PartitionKeyV2, PartitionHashRecordV2> execute() throws IgniteException {
+            try {
+                ignite.context().cache().context().database().waitForCheckpoint("VerifyBackupPartitions");
+            }
+            catch (IgniteCheckedException e) {
+                throw new IgniteException(
+                    "Failed to wait for checkpoint before executing verify backup partitions task", e);
+            }
+
             Set<Integer> grpIds = getGroupIds();
 
             completionCntr.set(0);
 
-            AtomicBoolean cpFlag = new AtomicBoolean();
+            List<Future<Map<PartitionKeyV2, PartitionHashRecordV2>>> partHashCalcFuts =
+                calcPartitionHashAsync(grpIds);
 
-            GridCacheDatabaseSharedManager db = null;
-
-            DbCheckpointListener lsnr = null;
-
-            if (arg.checkCrc() &&
-                ignite.context().cache().context().database() instanceof GridCacheDatabaseSharedManager) {
-                db = (GridCacheDatabaseSharedManager)ignite.context().cache().context().database();
-
-                lsnr = new DbCheckpointListener() {
-                    @Override public void onMarkCheckpointBegin(Context ctx) {
-                        /* No-op. */
-                    }
-
-                    @Override public void onCheckpointBegin(Context ctx) {
-                        if (ctx.hasPages())
-                            cpFlag.set(true);
-                    }
-
-                    @Override public void beforeCheckpointBegin(Context ctx) throws IgniteCheckedException {
-                        /* No-op. */
-                    }
-                };
-
-                db.addCheckpointListener(lsnr);
-            }
-
-            try {
-                if (arg.checkCrc() && IdleVerifyUtility.isCheckpointNow(db))
-                    throw new GridNotIdleException(IdleVerifyUtility.CLUSTER_NOT_IDLE_MSG);
+            Map<PartitionKeyV2, PartitionHashRecordV2> res = new HashMap<>();
 
-                List<Future<Map<PartitionKeyV2, PartitionHashRecordV2>>> partHashCalcFuts =
-                    calcPartitionHashAsync(grpIds, cpFlag);
+            List<IgniteException> exceptions = new ArrayList<>();
 
-                Map<PartitionKeyV2, PartitionHashRecordV2> res = new HashMap<>();
+            long lastProgressLogTs = U.currentTimeMillis();
 
-                List<IgniteException> exceptions = new ArrayList<>();
+            for (int i = 0; i < partHashCalcFuts.size(); ) {
+                Future<Map<PartitionKeyV2, PartitionHashRecordV2>> fut = partHashCalcFuts.get(i);
 
-                long lastProgressLogTs = U.currentTimeMillis();
-
-                for (int i = 0; i < partHashCalcFuts.size(); ) {
-                    Future<Map<PartitionKeyV2, PartitionHashRecordV2>> fut = partHashCalcFuts.get(i);
+                try {
+                    Map<PartitionKeyV2, PartitionHashRecordV2> partHash = fut.get(100, TimeUnit.MILLISECONDS);
 
-                    try {
-                        Map<PartitionKeyV2, PartitionHashRecordV2> partHash = fut.get(100, TimeUnit.MILLISECONDS);
+                    res.putAll(partHash);
 
-                        res.putAll(partHash);
+                    i++;
+                }
+                catch (InterruptedException | ExecutionException e) {
+                    if (e.getCause() instanceof IgniteException && !(e.getCause() instanceof GridNotIdleException)) {
+                        exceptions.add((IgniteException)e.getCause());
 
                         i++;
-                    }
-                    catch (InterruptedException | ExecutionException e) {
-                        if (e.getCause() instanceof IgniteException && !(e.getCause() instanceof GridNotIdleException)) {
-                            exceptions.add((IgniteException)e.getCause());
 
-                            i++;
-
-                            continue;
-                        }
+                        continue;
+                    }
 
-                        for (int j = i + 1; j < partHashCalcFuts.size(); j++)
-                            partHashCalcFuts.get(j).cancel(false);
+                    for (int j = i + 1; j < partHashCalcFuts.size(); j++)
+                        partHashCalcFuts.get(j).cancel(false);
 
-                        if (e instanceof InterruptedException)
-                            throw new IgniteInterruptedException((InterruptedException)e);
-                        else
-                            throw new IgniteException(e.getCause());
-                    }
-                    catch (TimeoutException ignored) {
-                        if (U.currentTimeMillis() - lastProgressLogTs > 3 * 60 * 1000L) {
-                            lastProgressLogTs = U.currentTimeMillis();
+                    if (e instanceof InterruptedException)
+                        throw new IgniteInterruptedException((InterruptedException)e);
+                    else
+                        throw new IgniteException(e.getCause());
+                }
+                catch (TimeoutException ignored) {
+                    if (U.currentTimeMillis() - lastProgressLogTs > 3 * 60 * 1000L) {
+                        lastProgressLogTs = U.currentTimeMillis();
 
-                            log.warning("idle_verify is still running, processed " + completionCntr.get() + " of " +
-                                partHashCalcFuts.size() + " local partitions");
-                        }
+                        log.warning("idle_verify is still running, processed " + completionCntr.get() + " of " +
+                            partHashCalcFuts.size() + " local partitions");
                     }
                 }
+            }
 
-                if (!F.isEmpty(exceptions))
-                    throw new IdleVerifyException(exceptions);
+            if (!F.isEmpty(exceptions))
+                throw new IdleVerifyException(exceptions);
 
-                return res;
-            }
-            finally {
-                if (db != null && lsnr != null)
-                    db.removeCheckpointListener(lsnr);
-            }
+            return res;
         }
 
         /**
@@ -388,8 +358,7 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
 
         /** */
         private List<Future<Map<PartitionKeyV2, PartitionHashRecordV2>>> calcPartitionHashAsync(
-            Set<Integer> grpIds,
-            AtomicBoolean cpFlag
+            Set<Integer> grpIds
         ) {
             List<Future<Map<PartitionKeyV2, PartitionHashRecordV2>>> partHashCalcFutures = new ArrayList<>();
 
@@ -402,7 +371,7 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
                 List<GridDhtLocalPartition> parts = grpCtx.topology().localPartitions();
 
                 for (GridDhtLocalPartition part : parts)
-                    partHashCalcFutures.add(calculatePartitionHashAsync(grpCtx, part, cpFlag));
+                    partHashCalcFutures.add(calculatePartitionHashAsync(grpCtx, part));
             }
 
             return partHashCalcFutures;
@@ -539,32 +508,31 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
         /**
          * @param grpCtx Group context.
          * @param part Local partition.
-         * @param cpFlag Checkpoint flag.
          */
         private Future<Map<PartitionKeyV2, PartitionHashRecordV2>> calculatePartitionHashAsync(
             final CacheGroupContext grpCtx,
-            final GridDhtLocalPartition part,
-            AtomicBoolean cpFlag
+            final GridDhtLocalPartition part
         ) {
-            return ForkJoinPool.commonPool().submit(() -> calculatePartitionHash(grpCtx, part, cpFlag));
+            return ForkJoinPool.commonPool().submit(() -> calculatePartitionHash(grpCtx, part));
         }
 
         /**
          * @param grpCtx Group context.
          * @param part Local partition.
-         * @param cpFlag Checkpoint flag.
          */
         private Map<PartitionKeyV2, PartitionHashRecordV2> calculatePartitionHash(
             CacheGroupContext grpCtx,
-            GridDhtLocalPartition part,
-            AtomicBoolean cpFlag
+            GridDhtLocalPartition part
         ) {
             if (!part.reserve())
                 return emptyMap();
 
             int partHash = 0;
             long partSize;
-            long updateCntrBefore = part.updateCounter();
+
+            @Nullable PartitionUpdateCounter updCntr = part.dataStore().partUpdateCounter();
+
+            PartitionUpdateCounter updateCntrBefore = updCntr == null ? null : updCntr.copy();
 
             PartitionKeyV2 partKey = new PartitionKeyV2(grpCtx.groupId(), part.id(), grpCtx.cacheOrGroupName());
 
@@ -579,7 +547,7 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
                         isPrimary,
                         consId,
                         partHash,
-                        updateCntrBefore,
+                        updateCntrBefore == null ? 0 : updateCntrBefore.get(),
                         part.state() == GridDhtPartitionState.MOVING ? PartitionHashRecordV2.MOVING_PARTITION_SIZE : 0,
                         part.state() == GridDhtPartitionState.MOVING ? PartitionState.MOVING : PartitionState.LOST
                     );
@@ -593,7 +561,7 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
                 partSize = part.dataStore().fullSize();
 
                 if (arg.checkCrc())
-                    checkPartitionCrc(grpCtx, part, cpFlag);
+                    checkPartitionCrc(grpCtx, part);
 
                 GridIterator<CacheDataRow> it = grpCtx.offheap().partitionIterator(part.id());
 
@@ -605,12 +573,12 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
                     partHash += Arrays.hashCode(row.value().valueBytes(grpCtx.cacheObjectContext()));
                 }
 
-                long updateCntrAfter = part.updateCounter();
+                PartitionUpdateCounter updateCntrAfter = part.dataStore().partUpdateCounter();
 
-                if (updateCntrBefore != updateCntrAfter) {
-                    throw new GridNotIdleException("Update counter of partition [grpId=" +
-                        grpCtx.groupId() + ", partId=" + part.id() + "] changed during hash calculation [before=" +
-                        updateCntrBefore + ", after=" + updateCntrAfter + "]");
+                if (updateCntrAfter != null && !updateCntrAfter.equals(updateCntrBefore)) {
+                    throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[grpName=" + grpCtx.cacheOrGroupName() +
+                        ", grpId=" + grpCtx.groupId() + ", partId=" + part.id() + "] changed during size " +
+                        "calculation [updCntrBefore=" + updateCntrBefore + ", updCntrAfter=" + updateCntrAfter + "]");
                 }
             }
             catch (IgniteCheckedException e) {
@@ -625,7 +593,8 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
             }
 
             PartitionHashRecordV2 partRec = new PartitionHashRecordV2(
-                partKey, isPrimary, consId, partHash, updateCntrBefore, partSize, PartitionState.OWNING
+                partKey, isPrimary, consId, partHash, updateCntrBefore == null ? 0 : updateCntrBefore.get(), partSize,
+                PartitionState.OWNING
             );
 
             completionCntr.incrementAndGet();
@@ -638,9 +607,8 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
          *
          * @param grpCtx Cache group context
          * @param part partition.
-         * @param cpFlag Checkpoint flag.
          */
-        private void checkPartitionCrc(CacheGroupContext grpCtx, GridDhtLocalPartition part, AtomicBoolean cpFlag) {
+        private void checkPartitionCrc(CacheGroupContext grpCtx, GridDhtLocalPartition part) {
             if (grpCtx.persistenceEnabled()) {
                 FilePageStore pageStore = null;
 
@@ -653,17 +621,14 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
 
                     pageStore = (FilePageStore)pageStoreMgr.getStore(grpCtx.groupId(), part.id());
 
-                    IdleVerifyUtility.checkPartitionsPageCrcSum(pageStore, grpCtx, part.id(), FLAG_DATA, cpFlag);
+                    IdleVerifyUtility.checkPartitionsPageCrcSum(pageStore, grpCtx, part.id(), FLAG_DATA);
                 }
                 catch (GridNotIdleException e) {
                     throw e;
                 }
                 catch (Exception | AssertionError e) {
-                    if (cpFlag.get())
-                        throw new GridNotIdleException("Checkpoint with dirty pages started! Cluster not idle!", e);
-
-                    String msg = new SB("CRC check of partition: ").a(part.id()).a(", for cache group ")
-                        .a(grpCtx.cacheOrGroupName()).a(" failed.")
+                    String msg = new SB("CRC check of partition: ").a(part.id()).a(", for cache group \"")
+                        .a(grpCtx.cacheOrGroupName()).a("\" failed.")
                         .a(pageStore != null ? " file: " + pageStore.getFileAbsolutePath() : "").toString();
 
                     log.error(msg, e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridArgumentCheck.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridArgumentCheck.java
index 529c8f1..2361c3b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridArgumentCheck.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridArgumentCheck.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.util;
 
 import java.util.Collection;
+
 import org.jetbrains.annotations.Nullable;
 
 /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesPartitionResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesPartitionResult.java
index 5d74a57..5ff666a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesPartitionResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesPartitionResult.java
@@ -23,38 +23,23 @@ import java.io.ObjectOutput;
 import java.util.ArrayList;
 import java.util.List;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorDataTransferObject;
 
 /**
- * Encapsulates intermediate results of validation of SQL index (if {@link #sqlIdxName} is present) or partition.
+ * Encapsulates intermediate results of validation of SQL index.
  */
 public class ValidateIndexesPartitionResult extends VisorDataTransferObject {
     /** */
     private static final long serialVersionUID = 0L;
 
-    /** Update counter. */
-    private long updateCntr;
-
-    /** Size. */
-    private long size;
-
-    /** Is primary. */
-    private boolean isPrimary;
-
-    /** Consistent id. */
-    @GridToStringInclude
-    private Object consistentId;
+    /** Max issues per result. */
+    private static final int MAX_ISSUES = 10;
 
     /** Issues. */
     @GridToStringExclude
-    private List<IndexValidationIssue> issues = new ArrayList<>(10);
-
-    /** Sql index name. */
-    @GridToStringExclude
-    private String sqlIdxName;
+    private List<IndexValidationIssue> issues = new ArrayList<>(MAX_ISSUES);
 
     /**
      *
@@ -64,50 +49,6 @@ public class ValidateIndexesPartitionResult extends VisorDataTransferObject {
     }
 
     /**
-     * @param updateCntr Update counter.
-     * @param size Size.
-     * @param isPrimary Is primary.
-     * @param consistentId Consistent id.
-     * @param sqlIdxName Sql index name (optional).
-     */
-    public ValidateIndexesPartitionResult(long updateCntr, long size, boolean isPrimary, Object consistentId,
-        String sqlIdxName) {
-        this.updateCntr = updateCntr;
-        this.size = size;
-        this.isPrimary = isPrimary;
-        this.consistentId = consistentId;
-        this.sqlIdxName = sqlIdxName;
-    }
-
-    /**
-     *
-     */
-    public long updateCntr() {
-        return updateCntr;
-    }
-
-    /**
-     *
-     */
-    public long size() {
-        return size;
-    }
-
-    /**
-     *
-     */
-    public boolean primary() {
-        return isPrimary;
-    }
-
-    /**
-     *
-     */
-    public Object consistentId() {
-        return consistentId;
-    }
-
-    /**
      *
      */
     public List<IndexValidationIssue> issues() {
@@ -115,18 +56,11 @@ public class ValidateIndexesPartitionResult extends VisorDataTransferObject {
     }
 
     /**
-     * @return <code>null</code> for partition validation result, SQL index name for index validation result
-     */
-    public String sqlIndexName() {
-        return sqlIdxName;
-    }
-
-    /**
      * @param t Issue.
      * @return True if there are already enough issues.
      */
     public boolean reportIssue(IndexValidationIssue t) {
-        if (issues.size() >= 10)
+        if (issues.size() == MAX_ISSUES)
             return true;
 
         issues.add(t);
@@ -141,30 +75,16 @@ public class ValidateIndexesPartitionResult extends VisorDataTransferObject {
 
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
-        out.writeLong(updateCntr);
-        out.writeLong(size);
-        out.writeBoolean(isPrimary);
-        out.writeObject(consistentId);
         U.writeCollection(out, issues);
-        U.writeString(out, sqlIdxName);
     }
 
     /** {@inheritDoc} */
     @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
-        updateCntr = in.readLong();
-        size = in.readLong();
-        isPrimary = in.readBoolean();
-        consistentId = in.readObject();
         issues = U.readList(in);
-
-        if (protoVer >= V2)
-            sqlIdxName = U.readString(in);
     }
 
     /** {@inheritDoc} */
     @Override public String toString() {
-        return sqlIdxName == null ? S.toString(ValidateIndexesPartitionResult.class, this) :
-            ValidateIndexesPartitionResult.class.getSimpleName() + " [consistentId=" + consistentId +
-                ", sqlIdxName=" + sqlIdxName + "]";
+        return S.toString(ValidateIndexesPartitionResult.class, this);
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java
index 1c19856..6bca524 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java
@@ -34,6 +34,7 @@ import java.util.stream.Collectors;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.AtomicConfiguration;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.ConnectorConfiguration;
@@ -53,8 +54,10 @@ import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.encryption.keystore.KeystoreEncryptionSpi;
 import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.testframework.junits.WithSystemProperty;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
 
 import static java.lang.String.join;
 import static java.lang.System.lineSeparator;
@@ -114,6 +117,27 @@ public abstract class GridCommandHandlerAbstractTest extends GridCommonAbstractT
     /** Last operation result. */
     protected Object lastOperationResult;
 
+    /** Persistence flag. */
+    private boolean persistent = true;
+
+    /**
+     * Persistence setter.
+     *
+     * @param pr {@code True} If persistence enable.
+     **/
+    protected void persistenceEnable(boolean pr) {
+        persistent = pr;
+    }
+
+    /**
+     * Persistence getter.
+     *
+     * @return Persistence enable flag.
+     */
+    protected boolean persistenceEnable() {
+        return persistent;
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
         super.beforeTestsStarted();
@@ -192,7 +216,7 @@ public abstract class GridCommandHandlerAbstractTest extends GridCommonAbstractT
             .setWalMode(WALMode.LOG_ONLY)
             .setCheckpointFrequency(checkpointFreq)
             .setDefaultDataRegionConfiguration(
-                new DataRegionConfiguration().setMaxSize(50L * 1024 * 1024).setPersistenceEnabled(true)
+                new DataRegionConfiguration().setMaxSize(50L * 1024 * 1024).setPersistenceEnabled(persistent)
             );
 
         if (dataRegionConfiguration != null)
@@ -376,17 +400,38 @@ public abstract class GridCommandHandlerAbstractTest extends GridCommonAbstractT
      *
      * @param ignite Ignite.
      * @param countEntries Count of entries.
+     * @param partitions Partitions count.
+     * @param filter Node filter.
      */
-    protected void createCacheAndPreload(Ignite ignite, int countEntries) {
+    protected void createCacheAndPreload(
+        Ignite ignite,
+        int countEntries,
+        int partitions,
+        @Nullable IgnitePredicate<ClusterNode> filter
+    ) {
         assert nonNull(ignite);
 
-        ignite.createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME)
-            .setAffinity(new RendezvousAffinityFunction(false, 32))
-            .setBackups(1)
-            .setEncryptionEnabled(encriptionEnabled));
+        CacheConfiguration<?, ?> ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+            .setAffinity(new RendezvousAffinityFunction(false, partitions))
+            .setBackups(1);
+
+        if (filter != null)
+            ccfg.setNodeFilter(filter);
+
+        ignite.createCache(ccfg);
 
         IgniteCache<Object, Object> cache = ignite.cache(DEFAULT_CACHE_NAME);
         for (int i = 0; i < countEntries; i++)
             cache.put(i, i);
     }
+
+    /**
+     * Creates default cache and preload some data entries.
+     *
+     * @param ignite Ignite.
+     * @param countEntries Count of entries.
+     */
+    protected void createCacheAndPreload(Ignite ignite, int countEntries) {
+        createCacheAndPreload(ignite, countEntries, 32, null);
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 7442ae0..b60a75a 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -44,6 +44,7 @@ import java.util.stream.IntStream;
 import javax.cache.processor.EntryProcessor;
 import javax.cache.processor.EntryProcessorException;
 import javax.cache.processor.MutableEntry;
+
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteAtomicSequence;
 import org.apache.ignite.IgniteCache;
@@ -118,6 +119,7 @@ import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_UNEXPECTED_ERROR;
 import static org.apache.ignite.internal.commandline.CommandList.DEACTIVATE;
 import static org.apache.ignite.internal.encryption.AbstractEncryptionTest.MASTER_KEY_NAME_2;
+import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.GRID_NOT_IDLE_MSG;
 import static org.apache.ignite.internal.processors.diagnostic.DiagnosticProcessor.DEFAULT_TARGET_FOLDER;
 import static org.apache.ignite.testframework.GridTestUtils.assertContains;
 import static org.apache.ignite.testframework.GridTestUtils.runAsync;
@@ -1240,7 +1242,7 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb
     /** */
     @Test
     public void testIdleVerifyCheckCrcFailsOnNotIdleCluster() throws Exception {
-        checkpointFreq = 100L;
+        checkpointFreq = 1000L;
 
         IgniteEx node = startGrids(2);
 
@@ -1258,7 +1260,7 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb
             ThreadLocalRandom rnd = ThreadLocalRandom.current();
 
             while (!stopFlag.get()) {
-                cache.put(rnd.nextInt(), rnd.nextInt());
+                cache.put(rnd.nextInt(1000), rnd.nextInt(1000));
 
                 if (Thread.interrupted())
                     break;
@@ -1275,6 +1277,8 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb
             assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--check-crc"));
         }
         finally {
+            doSleep(checkpointFreq);
+
             stopFlag.set(true);
 
             loadThread.join();
@@ -1289,7 +1293,7 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb
 
         String logFile = new String(Files.readAllBytes(new File(logFileName + ".txt").toPath()));
 
-        assertContains(log, logFile, "Checkpoint with dirty pages started! Cluster not idle!");
+        assertContains(log, logFile, GRID_NOT_IDLE_MSG);
     }
 
     /**
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
index 40f25cd..6f7e5fc 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
@@ -31,10 +31,10 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Supplier;
+
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteInterruptedException;
@@ -47,6 +47,7 @@ import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.PartitionUpdateCounter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.persistence.DbCheckpointListener;
@@ -72,6 +73,7 @@ import org.apache.ignite.internal.util.typedef.T3;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.LoggerResource;
 import org.h2.engine.Session;
@@ -88,7 +90,11 @@ import static java.util.Objects.nonNull;
 import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX;
 import static org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.GRID_NOT_IDLE_MSG;
+import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.compareUpdateCounters;
+import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.formatUpdateCountersDiff;
 import static org.apache.ignite.internal.util.IgniteUtils.error;
+import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.getUpdateCountersSnapshot;
 
 /**
  * Closure that locally validates indexes of given caches.
@@ -188,9 +194,11 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
     }
 
     /**
+     * Detect groups for further analysis.
      *
+     * @return Collection of group id.
      */
-    private VisorValidateIndexesJobResult call0() {
+    private Set<Integer> collectGroupIds() {
         Set<Integer> grpIds = new HashSet<>();
 
         Set<String> missingCaches = new HashSet<>();
@@ -199,46 +207,49 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
             for (String cacheName : cacheNames) {
                 DynamicCacheDescriptor desc = ignite.context().cache().cacheDescriptor(cacheName);
 
-                if (desc == null) {
+                if (desc == null)
                     missingCaches.add(cacheName);
-
-                    continue;
-                }
-
-                grpIds.add(desc.groupId());
+                else
+                    if (ignite.context().cache().cacheGroup(desc.groupId()).affinityNode())
+                        grpIds.add(desc.groupId());
             }
 
             if (!missingCaches.isEmpty()) {
-                StringBuilder strBuilder = new StringBuilder("The following caches do not exist: ");
-
-                for (String name : missingCaches)
-                    strBuilder.append(name).append(", ");
+                String errStr = "The following caches do not exist: " + String.join(", ", missingCaches);
 
-                strBuilder.delete(strBuilder.length() - 2, strBuilder.length());
-
-                throw new IgniteException(strBuilder.toString());
+                throw new IgniteException(errStr);
             }
         }
         else {
             Collection<CacheGroupContext> groups = ignite.context().cache().cacheGroups();
 
             for (CacheGroupContext grp : groups) {
-                if (!grp.systemCache() && !grp.isLocal())
+                if (!grp.systemCache() && !grp.isLocal() && grp.affinityNode())
                     grpIds.add(grp.groupId());
             }
         }
 
-        List<Future<Map<PartitionKey, ValidateIndexesPartitionResult>>> procPartFutures = new ArrayList<>();
-        List<Future<Map<String, ValidateIndexesPartitionResult>>> procIdxFutures = new ArrayList<>();
-        List<T3<CacheGroupContext, GridDhtLocalPartition, Future<CacheSize>>> cacheSizeFutures = new ArrayList<>();
-        List<T3<GridCacheContext, Index, Future<T2<Throwable, Long>>>> idxSizeFutures = new ArrayList<>();
+        return grpIds;
+    }
+
+    /**
+     *
+     */
+    private VisorValidateIndexesJobResult call0() {
+        Set<Integer> grpIds = collectGroupIds();
+
+        /** Update counters per partition per group. */
+        final Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrp =
+            getUpdateCountersSnapshot(ignite, grpIds);
+
+        IdleVerifyUtility.IdleChecker idleChecker = new IdleVerifyUtility.IdleChecker(ignite, partsWithCntrsPerGrp);
 
         List<T2<CacheGroupContext, GridDhtLocalPartition>> partArgs = new ArrayList<>();
         List<T2<GridCacheContext, Index>> idxArgs = new ArrayList<>();
 
         totalCacheGrps = grpIds.size();
 
-        Map<Integer, IndexIntegrityCheckIssue> integrityCheckResults = integrityCheckIndexesPartitions(grpIds);
+        Map<Integer, IndexIntegrityCheckIssue> integrityCheckResults = integrityCheckIndexesPartitions(grpIds, idleChecker);
 
         GridQueryProcessor qryProcessor = ignite.context().query();
         IgniteH2Indexing h2Indexing = (IgniteH2Indexing)qryProcessor.getIndexing();
@@ -283,11 +294,15 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
         totalPartitions = partArgs.size();
         totalIndexes = idxArgs.size();
 
-        for (T2<CacheGroupContext, GridDhtLocalPartition> t2 : partArgs)
-            procPartFutures.add(processPartitionAsync(t2.get1(), t2.get2()));
+        List<Future<Map<PartitionKey, ValidateIndexesPartitionResult>>> procPartFutures = new ArrayList<>(partArgs.size());
+        List<Future<Map<String, ValidateIndexesPartitionResult>>> procIdxFutures = new ArrayList<>(idxArgs.size());
+
+        List<T3<CacheGroupContext, GridDhtLocalPartition, Future<CacheSize>>> cacheSizeFutures = new ArrayList<>(partArgs.size());
+        List<T3<GridCacheContext, Index, Future<T2<Throwable, Long>>>> idxSizeFutures = new ArrayList<>(idxArgs.size());
 
-        for (T2<GridCacheContext, Index> t2 : idxArgs)
-            procIdxFutures.add(processIndexAsync(t2.get1(), t2.get2()));
+        partArgs.forEach(k -> procPartFutures.add(processPartitionAsync(k.get1(), k.get2())));
+
+        idxArgs.forEach(k -> procIdxFutures.add(processIndexAsync(k, idleChecker)));
 
         if (checkSizes) {
             for (T2<CacheGroupContext, GridDhtLocalPartition> partArg : partArgs) {
@@ -301,7 +316,7 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
                 GridCacheContext cacheCtx = idxArg.get1();
                 Index idx = idxArg.get2();
 
-                idxSizeFutures.add(new T3<>(cacheCtx, idx, calcIndexSizeAsync(cacheCtx, idx)));
+                idxSizeFutures.add(new T3<>(cacheCtx, idx, calcIndexSizeAsync(cacheCtx, idx, idleChecker)));
             }
         }
 
@@ -332,13 +347,27 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
                     idxResults.putAll(idxRes);
             }
 
-            for (; curCacheSize < cacheSizeFutures.size(); curCacheSize++)
-                cacheSizeFutures.get(curCacheSize).get3().get();
+            if (checkSizes) {
+                for (; curCacheSize < cacheSizeFutures.size(); curCacheSize++)
+                    cacheSizeFutures.get(curCacheSize).get3().get();
+
+                for (; curIdxSize < idxSizeFutures.size(); curIdxSize++)
+                    idxSizeFutures.get(curIdxSize).get3().get();
 
-            for (; curIdxSize < idxSizeFutures.size(); curIdxSize++)
-                idxSizeFutures.get(curIdxSize).get3().get();
+                checkSizes(cacheSizeFutures, idxSizeFutures, checkSizeResults);
 
-            checkSizes(cacheSizeFutures, idxSizeFutures, checkSizeResults);
+                Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrpAfterChecks =
+                    getUpdateCountersSnapshot(ignite, grpIds);
+
+                List<Integer> diff = compareUpdateCounters(ignite, partsWithCntrsPerGrp, partsWithCntrsPerGrpAfterChecks);
+
+                if (!F.isEmpty(diff)) {
+                    String res = formatUpdateCountersDiff(ignite, diff);
+
+                    if (!res.isEmpty())
+                        throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[" + res + "]");
+                }
+            }
 
             log.warning("ValidateIndexesClosure finished: processed " + totalPartitions + " partitions and "
                     + totalIndexes + " indexes.");
@@ -369,8 +398,14 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
 
     /**
      * @param grpIds Group ids.
+     * @param idleChecker Idle check closure.
      */
-    private Map<Integer, IndexIntegrityCheckIssue> integrityCheckIndexesPartitions(Set<Integer> grpIds) {
+    private Map<Integer, IndexIntegrityCheckIssue> integrityCheckIndexesPartitions(
+        Set<Integer> grpIds,
+        IgniteInClosure<Integer> idleChecker) {
+        if (!checkCrc)
+            return Collections.emptyMap();
+
         List<Future<T2<Integer, IndexIntegrityCheckIssue>>> integrityCheckFutures = new ArrayList<>(grpIds.size());
 
         Map<Integer, IndexIntegrityCheckIssue> integrityCheckResults = new HashMap<>();
@@ -382,51 +417,25 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
         DbCheckpointListener lsnr = null;
 
         try {
-            AtomicBoolean cpFlag = new AtomicBoolean();
-
-            if (db instanceof GridCacheDatabaseSharedManager) {
-                lsnr = new DbCheckpointListener() {
-                    @Override public void onMarkCheckpointBegin(Context ctx) {
-                        /* No-op. */
-                    }
-
-                    @Override public void onCheckpointBegin(Context ctx) {
-                        if (ctx.hasPages())
-                            cpFlag.set(true);
-                    }
-
-                    @Override public void beforeCheckpointBegin(Context ctx) throws IgniteCheckedException {
-                        /* No-op. */
-                    }
-                };
-
-                ((GridCacheDatabaseSharedManager)db).addCheckpointListener(lsnr);
-
-                if (IdleVerifyUtility.isCheckpointNow(db))
-                    throw new GridNotIdleException(IdleVerifyUtility.CLUSTER_NOT_IDLE_MSG);
-            }
+            for (Integer grpId: grpIds) {
+                final CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
 
-            if (checkCrc) {
-                for (Integer grpId : grpIds) {
-                    final CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
+                if (grpCtx == null || !grpCtx.persistenceEnabled()) {
+                    integrityCheckedIndexes.incrementAndGet();
 
-                    if (grpCtx == null || !grpCtx.persistenceEnabled()) {
-                        integrityCheckedIndexes.incrementAndGet();
-
-                        continue;
-                    }
+                    continue;
+                }
 
-                    Future<T2<Integer, IndexIntegrityCheckIssue>> checkFut =
+                Future<T2<Integer, IndexIntegrityCheckIssue>> checkFut =
                         calcExecutor.submit(new Callable<T2<Integer, IndexIntegrityCheckIssue>>() {
-                            @Override public T2<Integer, IndexIntegrityCheckIssue> call() throws Exception {
-                                IndexIntegrityCheckIssue issue = integrityCheckIndexPartition(grpCtx, cpFlag);
+                            @Override public T2<Integer, IndexIntegrityCheckIssue> call() {
+                                IndexIntegrityCheckIssue issue = integrityCheckIndexPartition(grpCtx, idleChecker);
 
                                 return new T2<>(grpCtx.groupId(), issue);
                             }
                         });
 
-                    integrityCheckFutures.add(checkFut);
-                }
+                integrityCheckFutures.add(checkFut);
             }
 
             for (Future<T2<Integer, IndexIntegrityCheckIssue>> fut : integrityCheckFutures) {
@@ -452,23 +461,25 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
 
     /**
      * @param gctx Cache group context.
-     * @param cpFlag Checkpoint status flag.
+     * @param idleChecker Idle check closure.
      */
-    private IndexIntegrityCheckIssue integrityCheckIndexPartition(CacheGroupContext gctx, AtomicBoolean cpFlag) {
+    private IndexIntegrityCheckIssue integrityCheckIndexPartition(
+        CacheGroupContext gctx,
+        IgniteInClosure<Integer> idleChecker
+    ) {
         GridKernalContext ctx = ignite.context();
         GridCacheSharedContext cctx = ctx.cache().context();
 
         try {
             FilePageStoreManager pageStoreMgr = (FilePageStoreManager)cctx.pageStore();
 
-            IdleVerifyUtility.checkPartitionsPageCrcSum(pageStoreMgr, gctx, INDEX_PARTITION, FLAG_IDX, cpFlag);
+            IdleVerifyUtility.checkPartitionsPageCrcSum(pageStoreMgr, gctx, INDEX_PARTITION, FLAG_IDX);
+
+            idleChecker.apply(gctx.groupId());
 
             return null;
         }
         catch (Throwable t) {
-            if (cpFlag.get())
-                throw new GridNotIdleException("Checkpoint with dirty pages started! Cluster not idle!", t);
-
             log.error("Integrity check of index partition of cache group " + gctx.cacheOrGroupName() + " failed", t);
 
             return new IndexIntegrityCheckIssue(gctx.cacheOrGroupName(), t);
@@ -486,11 +497,11 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
      * @param part Local partition.
      */
     private Future<Map<PartitionKey, ValidateIndexesPartitionResult>> processPartitionAsync(
-        final CacheGroupContext grpCtx,
-        final GridDhtLocalPartition part
+        CacheGroupContext grpCtx,
+        GridDhtLocalPartition part
     ) {
         return calcExecutor.submit(new Callable<Map<PartitionKey, ValidateIndexesPartitionResult>>() {
-            @Override public Map<PartitionKey, ValidateIndexesPartitionResult> call() throws Exception {
+            @Override public Map<PartitionKey, ValidateIndexesPartitionResult> call() {
                 return processPartition(grpCtx, part);
             }
         });
@@ -513,17 +524,13 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
             if (part.state() != OWNING)
                 return emptyMap();
 
-            long updateCntrBefore = part.updateCounter();
+            @Nullable PartitionUpdateCounter updCntr = part.dataStore().partUpdateCounter();
 
-            long partSize = part.dataStore().fullSize();
+            PartitionUpdateCounter updateCntrBefore = updCntr == null ? null : updCntr.copy();
 
             GridIterator<CacheDataRow> it = grpCtx.offheap().partitionIterator(part.id());
 
-            Object consId = ignite.context().discovery().localNode().consistentId();
-
-            boolean isPrimary = part.primary(grpCtx.topology().readyTopologyVersion());
-
-            partRes = new ValidateIndexesPartitionResult(updateCntrBefore, partSize, isPrimary, consId, null);
+            partRes = new ValidateIndexesPartitionResult();
 
             boolean enoughIssues = false;
 
@@ -630,6 +637,14 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
                     }
                 }
             }
+
+            PartitionUpdateCounter updateCntrAfter = part.dataStore().partUpdateCounter();
+
+            if (updateCntrAfter != null && !updateCntrAfter.equals(updateCntrBefore)) {
+                throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[grpName=" + grpCtx.cacheOrGroupName() +
+                    ", grpId=" + grpCtx.groupId() + ", partId=" + part.id() + "] changed during index validation " +
+                    "[before=" + updateCntrBefore + ", after=" + updateCntrAfter + "]");
+            }
         }
         catch (IgniteCheckedException e) {
             error(log, "Failed to process partition [grpId=" + grpCtx.groupId() +
@@ -674,26 +689,34 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
     }
 
     /**
-     * @param ctx Context.
-     * @param idx Index.
+     * @param cacheCtxWithIdx Cache context and appropriate index.
+     * @param idleChecker Idle check closure.
      */
-    private Future<Map<String, ValidateIndexesPartitionResult>> processIndexAsync(GridCacheContext ctx, Index idx) {
+    private Future<Map<String, ValidateIndexesPartitionResult>> processIndexAsync(
+        T2<GridCacheContext, Index> cacheCtxWithIdx,
+        IgniteInClosure<Integer> idleChecker
+    ) {
         return calcExecutor.submit(new Callable<Map<String, ValidateIndexesPartitionResult>>() {
-            @Override public Map<String, ValidateIndexesPartitionResult> call() throws Exception {
-                return processIndex(ctx, idx);
+            /** {@inheritDoc} */
+            @Override public Map<String, ValidateIndexesPartitionResult> call() {
+                return processIndex(cacheCtxWithIdx, idleChecker);
             }
         });
     }
 
     /**
-     * @param ctx Context.
-     * @param idx Index.
+     * @param cacheCtxWithIdx Cache context and appropriate index.
+     * @param idleChecker Idle check closure.
      */
-    private Map<String, ValidateIndexesPartitionResult> processIndex(GridCacheContext ctx, Index idx) {
-        Object consId = ignite.context().discovery().localNode().consistentId();
+    private Map<String, ValidateIndexesPartitionResult> processIndex(
+        T2<GridCacheContext, Index> cacheCtxWithIdx,
+        IgniteInClosure<Integer> idleChecker
+    ) {
+        GridCacheContext ctx = cacheCtxWithIdx.get1();
 
-        ValidateIndexesPartitionResult idxValidationRes = new ValidateIndexesPartitionResult(
-            -1, -1, true, consId, idx.getName());
+        Index idx = cacheCtxWithIdx.get2();
+
+        ValidateIndexesPartitionResult idxValidationRes = new ValidateIndexesPartitionResult();
 
         boolean enoughIssues = false;
 
@@ -765,7 +788,7 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
                     }
                 }
 
-                h2key = (KeyCacheObject)h2Row.key();
+                h2key = h2Row.key();
 
                 if (h2Row.link() != 0L) {
                     CacheDataRow cacheDataStoreRow = ctx.group().offheap().read(ctx, h2key);
@@ -803,6 +826,8 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
             idx.getName()
         );
 
+        idleChecker.apply(group.groupId());
+
         processedIndexes.incrementAndGet();
 
         printProgressOfIndexValidationIfNeeded();
@@ -839,6 +864,10 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
     ) {
         return calcExecutor.submit(() -> {
             try {
+                @Nullable PartitionUpdateCounter updCntr = locPart.dataStore().partUpdateCounter();
+
+                PartitionUpdateCounter updateCntrBefore = updCntr == null ? updCntr : updCntr.copy();
+
                 int grpId = grpCtx.groupId();
 
                 if (failCalcCacheSizeGrpIds.contains(grpId))
@@ -900,6 +929,14 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
                             .computeIfAbsent(tableName, s -> new AtomicLong()).incrementAndGet();
                     }
 
+                    PartitionUpdateCounter updateCntrAfter = locPart.dataStore().partUpdateCounter();
+
+                    if (updateCntrAfter != null && !updateCntrAfter.equals(updateCntrBefore)) {
+                        throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[grpName=" + grpCtx.cacheOrGroupName() +
+                            ", grpId=" + grpCtx.groupId() + ", partId=" + locPart.id() + "] changed during size " +
+                            "calculation [updCntrBefore=" + updateCntrBefore + ", updCntrAfter=" + updateCntrAfter + "]");
+                    }
+
                     return new CacheSize(null, cacheSizeByTbl);
                 }
                 catch (Throwable t) {
@@ -930,11 +967,13 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
      *
      * @param cacheCtx Cache context.
      * @param idx      Index.
+     * @param idleChecker Idle check closure.
      * @return Future with index size.
      */
     private Future<T2<Throwable, Long>> calcIndexSizeAsync(
         GridCacheContext cacheCtx,
-        Index idx
+        Index idx,
+        IgniteInClosure<Integer> idleChecker
     ) {
         return calcExecutor.submit(() -> {
             try {
@@ -947,6 +986,9 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
 
                 try {
                     long indexSize = ignite.context().query().getIndexing().indexSize(cacheName, tblName, idxName);
+
+                    idleChecker.apply(cacheCtx.groupId());
+
                     return new T2<>(null, indexSize);
                 }
                 catch (Throwable t) {
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/RebuildIndexTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/RebuildIndexTest.java
index b98bdf2..331e3a9 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/RebuildIndexTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/RebuildIndexTest.java
@@ -34,7 +34,7 @@ import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility;
+import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.verify.ValidateIndexesClosure;
 import org.apache.ignite.testframework.ListeningTestLogger;
@@ -176,8 +176,9 @@ public class RebuildIndexTest extends GridCommonAbstractTest {
 
         awaitPartitionMapExchange();
 
-        while (IdleVerifyUtility.isCheckpointNow(node.context().cache().context().database()))
-            doSleep(500);
+        forceCheckpoint();
+
+        enableCheckpoints(G.allGrids(), false);
 
         // Validate indexes on start.
         ValidateIndexesClosure clo = new ValidateIndexesClosure(Collections.singleton(CACHE_NAME), 0, 0, false, true);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
index 245787b..61ec646 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
@@ -20,19 +20,30 @@ package org.apache.ignite.util;
 import java.io.File;
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.cluster.ClusterState;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.junit.Test;
 
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME;
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
 import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_NAME;
+import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.GRID_NOT_IDLE_MSG;
 import static org.apache.ignite.testframework.GridTestUtils.assertContains;
 import static org.apache.ignite.testframework.GridTestUtils.assertNotContains;
+import static org.apache.ignite.testframework.GridTestUtils.runAsync;
 import static org.apache.ignite.util.GridCommandHandlerIndexingUtils.createAndFillCache;
 import static org.apache.ignite.util.GridCommandHandlerIndexingUtils.CACHE_NAME;
 import static org.apache.ignite.util.GridCommandHandlerIndexingUtils.GROUP_NAME;
@@ -86,8 +97,142 @@ public class GridCommandHandlerIndexingTest extends GridCommandHandlerClusterPer
 
         String out = testOut.toString();
 
-        assertContains(log, out, "Index validation failed");
-        assertContains(log, out, "Checkpoint with dirty pages started! Cluster not idle!");
+        assertContains(log, out, GRID_NOT_IDLE_MSG + "[\"" + GROUP_NAME + "\"]");
+    }
+
+
+    /** Run index validation check on busy cluster. */
+    @Test
+    public void testIdleVerifyCheckFailsOnNotIdleClusterWithOverwriteWithPers() throws Exception {
+        runIdleVerifyCheckCrcFailsOnNotIdleCluster(true);
+    }
+
+    /** Run index validation check on busy cluster. */
+    @Test
+    public void testIdleVerifyCheckFailsOnNotIdleClusterWithOverwriteWithoutPers() throws Exception {
+        persistenceEnable(false);
+
+        runIdleVerifyCheckCrcFailsOnNotIdleCluster(true);
+    }
+
+    /** Run index validation check on busy cluster. */
+    @Test
+    public void testIdleVerifyCheckFailsOnNotIdleClusterWithoutOverwriteWithPers() throws Exception {
+        runIdleVerifyCheckCrcFailsOnNotIdleCluster(false);
+    }
+
+    /** Run index validation check on busy cluster. */
+    @Test
+    public void testIdleVerifyCheckFailsOnNotIdleClusterWithoutOverwriteWithoutPers() throws Exception {
+        persistenceEnable(false);
+
+        runIdleVerifyCheckCrcFailsOnNotIdleCluster(false);
+    }
+
+    /**
+     * Check idle on busy cluster.
+     *
+     * @param allowOverwrite Overwrite param for datastreamer.
+     * @throws Exception
+     */
+    public void runIdleVerifyCheckCrcFailsOnNotIdleCluster(boolean allowOverwrite) throws Exception {
+        IgniteEx ig = startGrids(2);
+
+        ig.cluster().active(true);
+
+        int cntPreload = 100;
+
+        int maxItems = 100000;
+
+        createCacheAndPreload(ig, cntPreload, 1, new CachePredicate(F.asList(ig.name())));
+
+        if (persistenceEnable()) {
+            forceCheckpoint();
+
+            enableCheckpoints(G.allGrids(), false);
+        }
+
+        AtomicBoolean stopFlag = new AtomicBoolean();
+
+        CountDownLatch startLoading = new CountDownLatch(1);
+
+        IgniteInternalFuture f = runAsync(() -> {
+            try (IgniteDataStreamer<Object, Object> ldr = ig.dataStreamer(DEFAULT_CACHE_NAME)) {
+                ldr.allowOverwrite(allowOverwrite);
+
+                ldr.perThreadBufferSize(1);
+
+                boolean addFlag = true;
+
+                int i = cntPreload;
+
+                while (!stopFlag.get()) {
+                    if (addFlag)
+                        ldr.addData(i, i);
+                    else
+                        ldr.removeData(i);
+
+                    if (i == maxItems / 2)
+                        startLoading.countDown();
+
+                    if (i % 10 == 0)
+                        ldr.flush();
+
+                    if (++i == maxItems) {
+                        addFlag = !addFlag;
+
+                        i = 0;
+                    }
+                }
+            }
+        });
+
+        injectTestSystemOut();
+
+        startLoading.await();
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", "--check-crc", "--check-sizes"));
+
+        stopFlag.set(true);
+
+        f.get();
+
+        String out = testOut.toString();
+
+        assertContains(log, out, GRID_NOT_IDLE_MSG);
+
+        testOut.reset();
+
+        if (persistenceEnable())
+            enableCheckpoints(G.allGrids(), true);
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", "--check-crc", "--check-sizes"));
+
+        out = testOut.toString();
+
+        assertNotContains(log, out, GRID_NOT_IDLE_MSG);
+    }
+
+    /**
+     *
+     */
+    static class CachePredicate implements IgnitePredicate<ClusterNode> {
+        /** */
+        private List<String> excludeNodes;
+
+        /**
+         * @param excludeNodes Nodes names.
+         */
+        public CachePredicate(List<String> excludeNodes) {
+            this.excludeNodes = excludeNodes;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean apply(ClusterNode clusterNode) {
+            String name = clusterNode.attribute(ATTR_IGNITE_INSTANCE_NAME).toString();
+
+            return !excludeNodes.contains(name);
+        }
     }
 
     /**
@@ -161,7 +306,7 @@ public class GridCommandHandlerIndexingTest extends GridCommandHandlerClusterPer
     private Ignite prepareGridForTest() throws Exception {
         Ignite ignite = startGrids(GRID_CNT);
 
-        ignite.cluster().active(true);
+        ignite.cluster().state(ClusterState.ACTIVE);
 
         Ignite client = startGrid(CLIENT_NODE_NAME_PREFIX);