You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by as...@apache.org on 2020/05/19 07:27:12 UTC
[ignite] branch master updated: IGNITE-12938 Improve not idle
cluster detection for IdleVerify and ValidateIndexes. - Fixes #7743.
This is an automated email from the ASF dual-hosted git repository.
ascherbakov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 832f741 IGNITE-12938 Improve not idle cluster detection for IdleVerify and ValidateIndexes. - Fixes #7743.
832f741 is described below
commit 832f7414e336b85b497e9bbce235dc2d3e21d78c
Author: zstan <st...@gmail.com>
AuthorDate: Tue May 19 10:22:24 2020 +0300
IGNITE-12938 Improve not idle cluster detection for IdleVerify and ValidateIndexes. - Fixes #7743.
Signed-off-by: Aleksei Scherbakov <as...@apache.org>
---
.../processors/cache/PartitionUpdateCounter.java | 5 +
.../cache/PartitionUpdateCounterDebugWrapper.java | 5 +
.../cache/PartitionUpdateCounterTrackingImpl.java | 13 ++
.../cache/PartitionUpdateCounterVolatileImpl.java | 10 +
.../GridCacheDatabaseSharedManager.java | 20 +-
.../cache/persistence/pagemem/PageMemoryImpl.java | 3 +-
.../verify/CollectConflictPartitionKeysTask.java | 21 +-
.../cache/verify/IdleVerifyResultV2.java | 10 +-
.../processors/cache/verify/IdleVerifyUtility.java | 202 +++++++++++++++---
.../cache/verify/PartitionHashRecordV2.java | 12 +-
.../cache/verify/VerifyBackupPartitionsTask.java | 10 +-
.../cache/verify/VerifyBackupPartitionsTaskV2.java | 165 ++++++--------
.../ignite/internal/util/GridArgumentCheck.java | 1 +
.../verify/ValidateIndexesPartitionResult.java | 92 +-------
.../util/GridCommandHandlerAbstractTest.java | 57 ++++-
.../apache/ignite/util/GridCommandHandlerTest.java | 10 +-
.../visor/verify/ValidateIndexesClosure.java | 236 ++++++++++++---------
.../processors/database/RebuildIndexTest.java | 7 +-
.../util/GridCommandHandlerIndexingTest.java | 151 ++++++++++++-
19 files changed, 671 insertions(+), 359 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounter.java
index d0f0502..7b8f1c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounter.java
@@ -143,4 +143,9 @@ public interface PartitionUpdateCounter extends Iterable<long[]> {
* @return Cache group context.
*/
public CacheGroupContext context();
+
+ /**
+ * @return A deep copy of current instance.
+ */
+ public PartitionUpdateCounter copy();
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterDebugWrapper.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterDebugWrapper.java
index 556e329..2c57c77 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterDebugWrapper.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterDebugWrapper.java
@@ -268,6 +268,11 @@ public class PartitionUpdateCounterDebugWrapper implements PartitionUpdateCounte
}
/** {@inheritDoc} */
+ @Override public PartitionUpdateCounter copy() {
+ return new PartitionUpdateCounterDebugWrapper(partId, delegate.copy());
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
return delegate.toString();
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterTrackingImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterTrackingImpl.java
index 9a5f1b2..91927fb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterTrackingImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterTrackingImpl.java
@@ -454,4 +454,17 @@ public class PartitionUpdateCounterTrackingImpl implements PartitionUpdateCounte
@Override public CacheGroupContext context() {
return grp;
}
+
+ /** {@inheritDoc} */
+ @Override public PartitionUpdateCounter copy() {
+ PartitionUpdateCounterTrackingImpl copy = new PartitionUpdateCounterTrackingImpl(grp);
+
+ copy.cntr.set(cntr.get());
+ copy.first = first;
+ copy.queue = new TreeMap<>(queue);
+ copy.initCntr = initCntr;
+ copy.reserveCntr.set(reserveCntr.get());
+
+ return copy;
+ }
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterVolatileImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterVolatileImpl.java
index f25f456..ae32a51 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterVolatileImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounterVolatileImpl.java
@@ -167,4 +167,14 @@ public class PartitionUpdateCounterVolatileImpl implements PartitionUpdateCounte
@Override public CacheGroupContext context() {
return grp;
}
+
+ /** {@inheritDoc} */
+ @Override public PartitionUpdateCounter copy() {
+ PartitionUpdateCounterVolatileImpl copy = new PartitionUpdateCounterVolatileImpl(grp);
+
+ copy.cntr.set(cntr.get());
+ copy.initCntr = this.initCntr;
+
+ return copy;
+ }
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 7e2a0ae..792b62d 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -542,8 +542,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
int pagesNum = 0;
- boolean hasUserDirtyPages = false;
-
for (DataRegion reg : dataRegions()) {
if (!reg.config().isPersistenceEnabled())
continue;
@@ -560,7 +558,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
if (progress != null)
progress.currentCheckpointPagesCount(pagesNum);
- return new CheckpointPagesInfoHolder(res, pagesNum, hasUserDirtyPages);
+ return new CheckpointPagesInfoHolder(res, pagesNum);
}
/**
@@ -4010,8 +4008,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
dirtyPagesCount = cpPagesHolder.pagesNum();
- hasUserPages = !cpPagesHolder.onlySystemPages();
-
hasPartitionsToDestroy = !curr.getDestroyQueue().pendingReqs().isEmpty();
WALPointer cpPtr = null;
@@ -4042,7 +4038,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
tracker.onLockRelease();
}
- DbCheckpointListener.Context ctx = createOnCheckpointBeginContext(ctx0, hasUserPages);
+ DbCheckpointListener.Context ctx = createOnCheckpointBeginContext(ctx0, dirtyPagesCount > 0);
curr.transitTo(LOCK_RELEASED);
@@ -5582,9 +5578,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
/** Current checkpoint pages information. */
private static class CheckpointPagesInfoHolder {
- /** If {@code true} there are user pages in checkpoint. */
- private final boolean hasUserDirtyPages;
-
/** Total pages count in cp. */
private final int pagesNum;
@@ -5594,16 +5587,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
/** */
private CheckpointPagesInfoHolder(
Collection<Map.Entry<PageMemoryEx, GridMultiCollectionWrapper<FullPageId>>> pages,
- int num,
- boolean hasUserPages) {
+ int num) {
cpPages = pages;
pagesNum = num;
- hasUserDirtyPages = hasUserPages;
- }
-
- /** If {@code true} there are user pages in checkpoint. */
- private boolean onlySystemPages() {
- return !hasUserDirtyPages;
}
/** Total pages count in cp. */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index ff17a36..b17989cd 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -1119,7 +1119,7 @@ public class PageMemoryImpl implements PageMemoryEx {
IgniteInternalFuture allowToReplace
) throws IgniteException {
if (segments == null)
- return new GridMultiCollectionWrapper<>(Collections.<FullPageId>emptyList());
+ return new GridMultiCollectionWrapper<>(Collections.emptyList());
Collection[] collections = new Collection[segments.length];
@@ -1669,7 +1669,6 @@ public class PageMemoryImpl implements PageMemoryEx {
/**
* @param page Page pointer.
* @param fullId full page ID.
- * @param walPlc WAL policy
* @param walPlc Full page WAL record policy
* @param markDirty set dirty flag to page
* @param restore restore flag
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java
index 8e8128c..25d12c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java
@@ -37,6 +37,7 @@ import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.CacheObjectUtils;
import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.PartitionUpdateCounter;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
@@ -48,6 +49,8 @@ import org.apache.ignite.resources.LoggerResource;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
+import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.GRID_NOT_IDLE_MSG;
+
/**
*
*/
@@ -167,14 +170,16 @@ public class CollectConflictPartitionKeysTask extends ComputeTaskAdapter<Partiti
int partHash = 0;
long partSize;
- long updateCntrBefore;
+ PartitionUpdateCounter updateCntrBefore;
List<PartitionEntryHashRecord> partEntryHashRecords;
try {
if (part.state() != GridDhtPartitionState.OWNING)
return Collections.emptyMap();
- updateCntrBefore = part.updateCounter();
+ @Nullable PartitionUpdateCounter updCntr = part.dataStore().partUpdateCounter();
+
+ updateCntrBefore = updCntr == null ? null : updCntr.copy();
partSize = part.dataStore().fullSize();
@@ -202,12 +207,12 @@ public class CollectConflictPartitionKeysTask extends ComputeTaskAdapter<Partiti
row.key().valueBytes(grpCtx.cacheObjectContext()), row.version(), valHash));
}
- long updateCntrAfter = part.updateCounter();
+ PartitionUpdateCounter updateCntrAfter = part.dataStore().partUpdateCounter();
- if (updateCntrBefore != updateCntrAfter) {
- throw new IgniteException("Cluster is not idle: update counter of partition " + partKey.toString() +
- " changed during hash calculation [before=" + updateCntrBefore +
- ", after=" + updateCntrAfter + "]");
+ if (updateCntrAfter != null && !updateCntrAfter.equals(updateCntrBefore)) {
+ throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[grpName=" + grpCtx.cacheOrGroupName() +
+ ", grpId=" + grpCtx.groupId() + ", partId=" + part.id() + "] changed during hash calculation " +
+ "[before=" + updateCntrBefore + ", after=" + updateCntrAfter + "]");
}
}
catch (IgniteCheckedException e) {
@@ -224,7 +229,7 @@ public class CollectConflictPartitionKeysTask extends ComputeTaskAdapter<Partiti
boolean isPrimary = part.primary(grpCtx.topology().readyTopologyVersion());
PartitionHashRecord partHashRec = new PartitionHashRecord(
- partKey, isPrimary, consId, partHash, updateCntrBefore, partSize);
+ partKey, isPrimary, consId, partHash, updateCntrBefore == null ? 0 : updateCntrBefore.get(), partSize);
Map<PartitionHashRecord, List<PartitionEntryHashRecord>> res = new HashMap<>();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
index ece5b22..5d2a5d3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
@@ -24,6 +24,7 @@ import java.io.ObjectOutput;
import java.io.PrintWriter;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
@@ -143,7 +144,7 @@ public class IdleVerifyResultV2 extends VisorDataTransferObject {
* @return Moving partitions.
*/
public Map<PartitionKeyV2, List<PartitionHashRecordV2>> movingPartitions() {
- return movingPartitions;
+ return Collections.unmodifiableMap(movingPartitions);
}
/**
@@ -236,7 +237,12 @@ public class IdleVerifyResultV2 extends VisorDataTransferObject {
else
printConflicts(printer);
- printSkippedPartitions(printer, movingPartitions(), "MOVING");
+ Map<PartitionKeyV2, List<PartitionHashRecordV2>> moving = movingPartitions();
+
+ if (!moving.isEmpty())
+ printer.accept("Possible results are not full due to rebalance still in progress." + U.nl());
+
+ printSkippedPartitions(printer, moving, "MOVING");
printSkippedPartitions(printer, lostPartitions(), "LOST");
}
else {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java
index 10835f9..b93c122 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java
@@ -19,16 +19,28 @@ package org.apache.ignite.internal.processors.cache.verify;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.processors.cache.CacheGroupContext;
-import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointProgress;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.PartitionUpdateCounter;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.SB;
+import org.apache.ignite.lang.IgniteInClosure;
import org.jetbrains.annotations.Nullable;
/**
@@ -36,25 +48,24 @@ import org.jetbrains.annotations.Nullable;
*/
public class IdleVerifyUtility {
/** Cluster not idle message. */
- public static final String CLUSTER_NOT_IDLE_MSG = "Checkpoint with dirty pages started! Cluster not idle!";
+ public static final String GRID_NOT_IDLE_MSG =
+ "Cluster not idle. Modifications found in caches or groups: ";
/**
- * See {@link IdleVerifyUtility#checkPartitionsPageCrcSum(FilePageStore, CacheGroupContext, int, byte,
- * AtomicBoolean)}.
+ * See {@link IdleVerifyUtility#checkPartitionsPageCrcSum(FilePageStore, CacheGroupContext, int, byte)}.
*/
public static void checkPartitionsPageCrcSum(
@Nullable FilePageStoreManager pageStoreMgr,
CacheGroupContext grpCtx,
int partId,
- byte pageType,
- AtomicBoolean cpFlag
+ byte pageType
) throws IgniteCheckedException, GridNotIdleException {
if (!grpCtx.persistenceEnabled() || pageStoreMgr == null)
return;
FilePageStore pageStore = (FilePageStore)pageStoreMgr.getStore(grpCtx.groupId(), partId);
- checkPartitionsPageCrcSum(pageStore, grpCtx, partId, pageType, cpFlag);
+ checkPartitionsPageCrcSum(pageStore, grpCtx, partId, pageType);
}
/**
@@ -65,7 +76,6 @@ public class IdleVerifyUtility {
* @param grpCtx Passed cache group context.
* @param partId Partition id.
* @param pageType Page type. Possible types {@link PageIdAllocator#FLAG_DATA}, {@link PageIdAllocator#FLAG_IDX}.
- * @param cpFlag Checkpoint flag for detecting start checkpoint with dirty pages.
* @throws IgniteCheckedException If reading page failed.
* @throws GridNotIdleException If cluster not idle.
*/
@@ -73,8 +83,7 @@ public class IdleVerifyUtility {
FilePageStore pageStore,
CacheGroupContext grpCtx,
int partId,
- byte pageType,
- AtomicBoolean cpFlag
+ byte pageType
) throws IgniteCheckedException, GridNotIdleException {
assert pageType == PageIdAllocator.FLAG_DATA || pageType == PageIdAllocator.FLAG_IDX : pageType;
@@ -87,28 +96,169 @@ public class IdleVerifyUtility {
for (int pageNo = 0; pageNo < pageStore.pages(); pageId++, pageNo++) {
buf.clear();
- if (cpFlag.get())
- throw new GridNotIdleException(CLUSTER_NOT_IDLE_MSG);
-
pageStore.read(pageId, buf, true);
}
+ }
+
+ /**
+ * Gather updateCounters info.
+ * Holds {@link org.apache.ignite.internal.processors.cache.PartitionUpdateCounter#copy} of update counters.
+ *
+ * @param ign Ignite instance.
+ * @param grpIds Group Id`s.
+ * @return Current groups distribution with update counters per partitions.
+ */
+ public static Map<Integer, Map<Integer, PartitionUpdateCounter>> getUpdateCountersSnapshot(
+ IgniteEx ign,
+ Set<Integer> grpIds
+ ) {
+ Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCountersPerGrp = new HashMap<>();
+
+ for (Integer grpId : grpIds) {
+ CacheGroupContext grpCtx = ign.context().cache().cacheGroup(grpId);
+
+ if (grpCtx == null)
+ throw new GridNotIdleException("Group not found: " + grpId + "."
+ + " Possible reasons: rebalance in progress or concurrent cache destroy.");
+
+ GridDhtPartitionTopology top = grpCtx.topology();
+
+ Map<Integer, PartitionUpdateCounter> partsWithCounters =
+ partsWithCountersPerGrp.computeIfAbsent(grpId, k -> new HashMap<>());
+
+ for (GridDhtLocalPartition part : top.currentLocalPartitions()) {
+ if (part.state() != GridDhtPartitionState.OWNING)
+ continue;
+
+ @Nullable PartitionUpdateCounter updCntr = part.dataStore().partUpdateCounter();
+
+ partsWithCounters.put(part.id(), updCntr == null ? null : updCntr.copy());
+ }
+ }
+
+ return partsWithCountersPerGrp;
+ }
+
+ /**
+ * Prints diff between incoming update counters snapshots.
+ *
+ * @param ig Ignite instance.
+ * @param diff Compared groups diff.
+ * @return Formatted diff representation.
+ */
+ public static String formatUpdateCountersDiff(IgniteEx ig, List<Integer> diff) {
+ SB sb = null;
+
+ if (!diff.isEmpty()) {
+ sb = new SB();
+
+ for (int grpId0 : diff) {
+ if (sb.length() != 0)
+ sb.a(", ");
+ else
+ sb.a("\"");
+
+ DynamicCacheDescriptor desc = ig.context().cache().cacheDescriptor(grpId0);
+
+ CacheGroupContext grpCtx = ig.context().cache().cacheGroup(desc == null ? grpId0 : desc.groupId());
+
+ sb.a(grpCtx.cacheOrGroupName());
+ }
+
+ sb.a("\"");
+ }
+
+ return sb != null ? sb.toString() : "";
+ }
+
+ /**
+ * Compares two sets with partitions and upd counters per group distribution.
+ *
+ * @param ign Ignite instance.
+ * @param cntrsIn Group id`s with counters per partitions per groups distribution.
+ * @param grpId Group id to compare.
+ * @return Diff with grpId info between two sets.
+ */
+ public static List<Integer> compareUpdateCounters(
+ IgniteEx ign,
+ Map<Integer, Map<Integer, PartitionUpdateCounter>> cntrsIn,
+ Integer grpId
+ ) {
+ Map<Integer, Map<Integer, PartitionUpdateCounter>> curCntrs =
+ getUpdateCountersSnapshot(ign, Collections.singleton(grpId));
- if (cpFlag.get())
- throw new GridNotIdleException(CLUSTER_NOT_IDLE_MSG);
+ if (curCntrs.isEmpty())
+ throw new GridNotIdleException("No OWNING partitions for group: " + grpId);
+
+ return compareUpdateCounters(ign, cntrsIn, curCntrs);
}
/**
- * @param db Shared DB manager.
- * @return {@code True} if checkpoint is now, {@code False} otherwise.
+ * Compares two sets with partitions and upd counters per group distribution.
+ *
+ * @param ign Ignite instance.
+ * @param cntrsEth Ethalon group id`s with counters per partitions per groups distribution.
+ * @param curCntrs Group id`s with counters per partitions per groups distribution compare with.
+ * @return Diff with grpId info between two sets.
*/
- public static boolean isCheckpointNow(@Nullable IgniteCacheDatabaseSharedManager db) {
- if (!(db instanceof GridCacheDatabaseSharedManager))
- return false;
+ public static List<Integer> compareUpdateCounters(
+ IgniteEx ign,
+ Map<Integer, Map<Integer, PartitionUpdateCounter>> cntrsEth,
+ Map<Integer, Map<Integer, PartitionUpdateCounter>> curCntrs
+ ) {
+ List<Integer> diff = new ArrayList<>();
+
+ Integer grpId;
+ for (Map.Entry<Integer, Map<Integer, PartitionUpdateCounter>> curEntry : curCntrs.entrySet()) {
+ Map<Integer, PartitionUpdateCounter> partsWithCntrsCur = curEntry.getValue();
+
+ grpId = curEntry.getKey();
- CheckpointProgress progress =
- ((GridCacheDatabaseSharedManager)db).getCheckpointer().currentProgress();
+ if (partsWithCntrsCur == null)
+ throw new GridNotIdleException("Group not found: " + grpId + "."
+ + " Possible reasons: rebalance in progress or concurrent cache destroy.");
+
+ Map<Integer, PartitionUpdateCounter> partsWithCntrsIn = cntrsEth.get(grpId);
+
+ if (!partsWithCntrsIn.equals(partsWithCntrsCur))
+ diff.add(grpId);
+ }
- return progress != null && progress.inProgress();
+ return diff;
+ }
+
+ /**
+ * Idle checker.
+ */
+ public static class IdleChecker implements IgniteInClosure<Integer> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private final IgniteEx ig;
+
+ /** Group id`s snapshot with partitions and counters distrubution. */
+ private final Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrp;
+
+ /** */
+ public IdleChecker(IgniteEx ig, Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrp) {
+ this.ig = ig;
+ this.partsWithCntrsPerGrp = partsWithCntrsPerGrp;
+ }
+
+ /** */
+ @Override public void apply(Integer grpId) {
+ List<Integer> diff;
+
+ diff = compareUpdateCounters(ig, partsWithCntrsPerGrp, grpId);
+
+ if (!F.isEmpty(diff)) {
+ String res = formatUpdateCountersDiff(ig, diff);
+
+ if (!res.isEmpty())
+ throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[" + res + "]");
+ }
+ }
}
/** */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecordV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecordV2.java
index 5cefa66..ce1d1c3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecordV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecordV2.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.cache.verify;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
+
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -70,8 +71,15 @@ public class PartitionHashRecordV2 extends VisorDataTransferObject {
* @param size Size.
* @param partitionState Partition state.
*/
- public PartitionHashRecordV2(PartitionKeyV2 partKey, boolean isPrimary,
- Object consistentId, int partHash, long updateCntr, long size, PartitionState partitionState) {
+ public PartitionHashRecordV2(
+ PartitionKeyV2 partKey,
+ boolean isPrimary,
+ Object consistentId,
+ int partHash,
+ long updateCntr,
+ long size,
+ PartitionState partitionState
+ ) {
this.partKey = partKey;
this.isPrimary = isPrimary;
this.consistentId = consistentId;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
index f700567..67b1146 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
@@ -56,6 +56,8 @@ import org.apache.ignite.resources.LoggerResource;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
+import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.GRID_NOT_IDLE_MSG;
+
/**
* Task for comparing update counters and checksums between primary and backup partitions of specified caches.
* <br>
@@ -103,7 +105,7 @@ public class VerifyBackupPartitionsTask extends ComputeTaskAdapter<Set<String>,
for (Map.Entry<PartitionKey, PartitionHashRecord> e : nodeHashes.entrySet()) {
if (!clusterHashes.containsKey(e.getKey()))
- clusterHashes.put(e.getKey(), new ArrayList<PartitionHashRecord>());
+ clusterHashes.put(e.getKey(), new ArrayList<>());
clusterHashes.get(e.getKey()).add(e.getValue());
}
@@ -330,9 +332,9 @@ public class VerifyBackupPartitionsTask extends ComputeTaskAdapter<Set<String>,
long updateCntrAfter = part.updateCounter();
if (updateCntrBefore != updateCntrAfter) {
- throw new IgniteException("Cluster is not idle: update counter of partition [grpId=" +
- grpCtx.groupId() + ", partId=" + part.id() + "] changed during hash calculation [before=" +
- updateCntrBefore + ", after=" + updateCntrAfter + "]");
+ throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[grpName=" + grpCtx.cacheOrGroupName() +
+ ", grpId=" + grpCtx.groupId() + ", partId=" + part.id() + "] changed during hash calculation " +
+ "[before=" + updateCntrBefore + ", after=" + updateCntrAfter + "]");
}
}
catch (IgniteCheckedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
index 187dc8ce..ed405a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
@@ -30,7 +30,6 @@ import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import org.apache.ignite.IgniteCheckedException;
@@ -50,11 +49,10 @@ import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
import org.apache.ignite.internal.processors.cache.GridCacheUtils;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.PartitionUpdateCounter;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
-import org.apache.ignite.internal.processors.cache.persistence.DbCheckpointListener;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecordV2.PartitionState;
@@ -74,6 +72,7 @@ import org.jetbrains.annotations.Nullable;
import static java.util.Collections.emptyMap;
import static org.apache.ignite.cache.CacheMode.LOCAL;
import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA;
+import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.GRID_NOT_IDLE_MSG;
/**
* Task for comparing update counters and checksums between primary and backup partitions of specified caches.
@@ -252,97 +251,68 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
/** {@inheritDoc} */
@Override public Map<PartitionKeyV2, PartitionHashRecordV2> execute() throws IgniteException {
+ try {
+ ignite.context().cache().context().database().waitForCheckpoint("VerifyBackupPartitions");
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(
+ "Failed to wait for checkpoint before executing verify backup partitions task", e);
+ }
+
Set<Integer> grpIds = getGroupIds();
completionCntr.set(0);
- AtomicBoolean cpFlag = new AtomicBoolean();
+ List<Future<Map<PartitionKeyV2, PartitionHashRecordV2>>> partHashCalcFuts =
+ calcPartitionHashAsync(grpIds);
- GridCacheDatabaseSharedManager db = null;
-
- DbCheckpointListener lsnr = null;
-
- if (arg.checkCrc() &&
- ignite.context().cache().context().database() instanceof GridCacheDatabaseSharedManager) {
- db = (GridCacheDatabaseSharedManager)ignite.context().cache().context().database();
-
- lsnr = new DbCheckpointListener() {
- @Override public void onMarkCheckpointBegin(Context ctx) {
- /* No-op. */
- }
-
- @Override public void onCheckpointBegin(Context ctx) {
- if (ctx.hasPages())
- cpFlag.set(true);
- }
-
- @Override public void beforeCheckpointBegin(Context ctx) throws IgniteCheckedException {
- /* No-op. */
- }
- };
-
- db.addCheckpointListener(lsnr);
- }
-
- try {
- if (arg.checkCrc() && IdleVerifyUtility.isCheckpointNow(db))
- throw new GridNotIdleException(IdleVerifyUtility.CLUSTER_NOT_IDLE_MSG);
+ Map<PartitionKeyV2, PartitionHashRecordV2> res = new HashMap<>();
- List<Future<Map<PartitionKeyV2, PartitionHashRecordV2>>> partHashCalcFuts =
- calcPartitionHashAsync(grpIds, cpFlag);
+ List<IgniteException> exceptions = new ArrayList<>();
- Map<PartitionKeyV2, PartitionHashRecordV2> res = new HashMap<>();
+ long lastProgressLogTs = U.currentTimeMillis();
- List<IgniteException> exceptions = new ArrayList<>();
+ for (int i = 0; i < partHashCalcFuts.size(); ) {
+ Future<Map<PartitionKeyV2, PartitionHashRecordV2>> fut = partHashCalcFuts.get(i);
- long lastProgressLogTs = U.currentTimeMillis();
-
- for (int i = 0; i < partHashCalcFuts.size(); ) {
- Future<Map<PartitionKeyV2, PartitionHashRecordV2>> fut = partHashCalcFuts.get(i);
+ try {
+ Map<PartitionKeyV2, PartitionHashRecordV2> partHash = fut.get(100, TimeUnit.MILLISECONDS);
- try {
- Map<PartitionKeyV2, PartitionHashRecordV2> partHash = fut.get(100, TimeUnit.MILLISECONDS);
+ res.putAll(partHash);
- res.putAll(partHash);
+ i++;
+ }
+ catch (InterruptedException | ExecutionException e) {
+ if (e.getCause() instanceof IgniteException && !(e.getCause() instanceof GridNotIdleException)) {
+ exceptions.add((IgniteException)e.getCause());
i++;
- }
- catch (InterruptedException | ExecutionException e) {
- if (e.getCause() instanceof IgniteException && !(e.getCause() instanceof GridNotIdleException)) {
- exceptions.add((IgniteException)e.getCause());
- i++;
-
- continue;
- }
+ continue;
+ }
- for (int j = i + 1; j < partHashCalcFuts.size(); j++)
- partHashCalcFuts.get(j).cancel(false);
+ for (int j = i + 1; j < partHashCalcFuts.size(); j++)
+ partHashCalcFuts.get(j).cancel(false);
- if (e instanceof InterruptedException)
- throw new IgniteInterruptedException((InterruptedException)e);
- else
- throw new IgniteException(e.getCause());
- }
- catch (TimeoutException ignored) {
- if (U.currentTimeMillis() - lastProgressLogTs > 3 * 60 * 1000L) {
- lastProgressLogTs = U.currentTimeMillis();
+ if (e instanceof InterruptedException)
+ throw new IgniteInterruptedException((InterruptedException)e);
+ else
+ throw new IgniteException(e.getCause());
+ }
+ catch (TimeoutException ignored) {
+ if (U.currentTimeMillis() - lastProgressLogTs > 3 * 60 * 1000L) {
+ lastProgressLogTs = U.currentTimeMillis();
- log.warning("idle_verify is still running, processed " + completionCntr.get() + " of " +
- partHashCalcFuts.size() + " local partitions");
- }
+ log.warning("idle_verify is still running, processed " + completionCntr.get() + " of " +
+ partHashCalcFuts.size() + " local partitions");
}
}
+ }
- if (!F.isEmpty(exceptions))
- throw new IdleVerifyException(exceptions);
+ if (!F.isEmpty(exceptions))
+ throw new IdleVerifyException(exceptions);
- return res;
- }
- finally {
- if (db != null && lsnr != null)
- db.removeCheckpointListener(lsnr);
- }
+ return res;
}
/**
@@ -388,8 +358,7 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
/** */
private List<Future<Map<PartitionKeyV2, PartitionHashRecordV2>>> calcPartitionHashAsync(
- Set<Integer> grpIds,
- AtomicBoolean cpFlag
+ Set<Integer> grpIds
) {
List<Future<Map<PartitionKeyV2, PartitionHashRecordV2>>> partHashCalcFutures = new ArrayList<>();
@@ -402,7 +371,7 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
List<GridDhtLocalPartition> parts = grpCtx.topology().localPartitions();
for (GridDhtLocalPartition part : parts)
- partHashCalcFutures.add(calculatePartitionHashAsync(grpCtx, part, cpFlag));
+ partHashCalcFutures.add(calculatePartitionHashAsync(grpCtx, part));
}
return partHashCalcFutures;
@@ -539,32 +508,31 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
/**
* @param grpCtx Group context.
* @param part Local partition.
- * @param cpFlag Checkpoint flag.
*/
private Future<Map<PartitionKeyV2, PartitionHashRecordV2>> calculatePartitionHashAsync(
final CacheGroupContext grpCtx,
- final GridDhtLocalPartition part,
- AtomicBoolean cpFlag
+ final GridDhtLocalPartition part
) {
- return ForkJoinPool.commonPool().submit(() -> calculatePartitionHash(grpCtx, part, cpFlag));
+ return ForkJoinPool.commonPool().submit(() -> calculatePartitionHash(grpCtx, part));
}
/**
* @param grpCtx Group context.
* @param part Local partition.
- * @param cpFlag Checkpoint flag.
*/
private Map<PartitionKeyV2, PartitionHashRecordV2> calculatePartitionHash(
CacheGroupContext grpCtx,
- GridDhtLocalPartition part,
- AtomicBoolean cpFlag
+ GridDhtLocalPartition part
) {
if (!part.reserve())
return emptyMap();
int partHash = 0;
long partSize;
- long updateCntrBefore = part.updateCounter();
+
+ @Nullable PartitionUpdateCounter updCntr = part.dataStore().partUpdateCounter();
+
+ PartitionUpdateCounter updateCntrBefore = updCntr == null ? null : updCntr.copy();
PartitionKeyV2 partKey = new PartitionKeyV2(grpCtx.groupId(), part.id(), grpCtx.cacheOrGroupName());
@@ -579,7 +547,7 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
isPrimary,
consId,
partHash,
- updateCntrBefore,
+ updateCntrBefore == null ? 0 : updateCntrBefore.get(),
part.state() == GridDhtPartitionState.MOVING ? PartitionHashRecordV2.MOVING_PARTITION_SIZE : 0,
part.state() == GridDhtPartitionState.MOVING ? PartitionState.MOVING : PartitionState.LOST
);
@@ -593,7 +561,7 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
partSize = part.dataStore().fullSize();
if (arg.checkCrc())
- checkPartitionCrc(grpCtx, part, cpFlag);
+ checkPartitionCrc(grpCtx, part);
GridIterator<CacheDataRow> it = grpCtx.offheap().partitionIterator(part.id());
@@ -605,12 +573,12 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
partHash += Arrays.hashCode(row.value().valueBytes(grpCtx.cacheObjectContext()));
}
- long updateCntrAfter = part.updateCounter();
+ PartitionUpdateCounter updateCntrAfter = part.dataStore().partUpdateCounter();
- if (updateCntrBefore != updateCntrAfter) {
- throw new GridNotIdleException("Update counter of partition [grpId=" +
- grpCtx.groupId() + ", partId=" + part.id() + "] changed during hash calculation [before=" +
- updateCntrBefore + ", after=" + updateCntrAfter + "]");
+ if (updateCntrAfter != null && !updateCntrAfter.equals(updateCntrBefore)) {
+ throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[grpName=" + grpCtx.cacheOrGroupName() +
+ ", grpId=" + grpCtx.groupId() + ", partId=" + part.id() + "] changed during size " +
+ "calculation [updCntrBefore=" + updateCntrBefore + ", updCntrAfter=" + updateCntrAfter + "]");
}
}
catch (IgniteCheckedException e) {
@@ -625,7 +593,8 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
}
PartitionHashRecordV2 partRec = new PartitionHashRecordV2(
- partKey, isPrimary, consId, partHash, updateCntrBefore, partSize, PartitionState.OWNING
+ partKey, isPrimary, consId, partHash, updateCntrBefore == null ? 0 : updateCntrBefore.get(), partSize,
+ PartitionState.OWNING
);
completionCntr.incrementAndGet();
@@ -638,9 +607,8 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
*
* @param grpCtx Cache group context
* @param part partition.
- * @param cpFlag Checkpoint flag.
*/
- private void checkPartitionCrc(CacheGroupContext grpCtx, GridDhtLocalPartition part, AtomicBoolean cpFlag) {
+ private void checkPartitionCrc(CacheGroupContext grpCtx, GridDhtLocalPartition part) {
if (grpCtx.persistenceEnabled()) {
FilePageStore pageStore = null;
@@ -653,17 +621,14 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
pageStore = (FilePageStore)pageStoreMgr.getStore(grpCtx.groupId(), part.id());
- IdleVerifyUtility.checkPartitionsPageCrcSum(pageStore, grpCtx, part.id(), FLAG_DATA, cpFlag);
+ IdleVerifyUtility.checkPartitionsPageCrcSum(pageStore, grpCtx, part.id(), FLAG_DATA);
}
catch (GridNotIdleException e) {
throw e;
}
catch (Exception | AssertionError e) {
- if (cpFlag.get())
- throw new GridNotIdleException("Checkpoint with dirty pages started! Cluster not idle!", e);
-
- String msg = new SB("CRC check of partition: ").a(part.id()).a(", for cache group ")
- .a(grpCtx.cacheOrGroupName()).a(" failed.")
+ String msg = new SB("CRC check of partition: ").a(part.id()).a(", for cache group \"")
+ .a(grpCtx.cacheOrGroupName()).a("\" failed.")
.a(pageStore != null ? " file: " + pageStore.getFileAbsolutePath() : "").toString();
log.error(msg, e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridArgumentCheck.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridArgumentCheck.java
index 529c8f1..2361c3b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridArgumentCheck.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridArgumentCheck.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.util;
import java.util.Collection;
+
import org.jetbrains.annotations.Nullable;
/**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesPartitionResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesPartitionResult.java
index 5d74a57..5ff666a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesPartitionResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesPartitionResult.java
@@ -23,38 +23,23 @@ import java.io.ObjectOutput;
import java.util.ArrayList;
import java.util.List;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.internal.visor.VisorDataTransferObject;
/**
- * Encapsulates intermediate results of validation of SQL index (if {@link #sqlIdxName} is present) or partition.
+ * Encapsulates intermediate results of validation of SQL index.
*/
public class ValidateIndexesPartitionResult extends VisorDataTransferObject {
/** */
private static final long serialVersionUID = 0L;
- /** Update counter. */
- private long updateCntr;
-
- /** Size. */
- private long size;
-
- /** Is primary. */
- private boolean isPrimary;
-
- /** Consistent id. */
- @GridToStringInclude
- private Object consistentId;
+ /** Max issues per result. */
+ private static final int MAX_ISSUES = 10;
/** Issues. */
@GridToStringExclude
- private List<IndexValidationIssue> issues = new ArrayList<>(10);
-
- /** Sql index name. */
- @GridToStringExclude
- private String sqlIdxName;
+ private List<IndexValidationIssue> issues = new ArrayList<>(MAX_ISSUES);
/**
*
@@ -64,50 +49,6 @@ public class ValidateIndexesPartitionResult extends VisorDataTransferObject {
}
/**
- * @param updateCntr Update counter.
- * @param size Size.
- * @param isPrimary Is primary.
- * @param consistentId Consistent id.
- * @param sqlIdxName Sql index name (optional).
- */
- public ValidateIndexesPartitionResult(long updateCntr, long size, boolean isPrimary, Object consistentId,
- String sqlIdxName) {
- this.updateCntr = updateCntr;
- this.size = size;
- this.isPrimary = isPrimary;
- this.consistentId = consistentId;
- this.sqlIdxName = sqlIdxName;
- }
-
- /**
- *
- */
- public long updateCntr() {
- return updateCntr;
- }
-
- /**
- *
- */
- public long size() {
- return size;
- }
-
- /**
- *
- */
- public boolean primary() {
- return isPrimary;
- }
-
- /**
- *
- */
- public Object consistentId() {
- return consistentId;
- }
-
- /**
*
*/
public List<IndexValidationIssue> issues() {
@@ -115,18 +56,11 @@ public class ValidateIndexesPartitionResult extends VisorDataTransferObject {
}
/**
- * @return <code>null</code> for partition validation result, SQL index name for index validation result
- */
- public String sqlIndexName() {
- return sqlIdxName;
- }
-
- /**
* @param t Issue.
* @return True if there are already enough issues.
*/
public boolean reportIssue(IndexValidationIssue t) {
- if (issues.size() >= 10)
+ if (issues.size() == MAX_ISSUES)
return true;
issues.add(t);
@@ -141,30 +75,16 @@ public class ValidateIndexesPartitionResult extends VisorDataTransferObject {
/** {@inheritDoc} */
@Override protected void writeExternalData(ObjectOutput out) throws IOException {
- out.writeLong(updateCntr);
- out.writeLong(size);
- out.writeBoolean(isPrimary);
- out.writeObject(consistentId);
U.writeCollection(out, issues);
- U.writeString(out, sqlIdxName);
}
/** {@inheritDoc} */
@Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
- updateCntr = in.readLong();
- size = in.readLong();
- isPrimary = in.readBoolean();
- consistentId = in.readObject();
issues = U.readList(in);
-
- if (protoVer >= V2)
- sqlIdxName = U.readString(in);
}
/** {@inheritDoc} */
@Override public String toString() {
- return sqlIdxName == null ? S.toString(ValidateIndexesPartitionResult.class, this) :
- ValidateIndexesPartitionResult.class.getSimpleName() + " [consistentId=" + consistentId +
- ", sqlIdxName=" + sqlIdxName + "]";
+ return S.toString(ValidateIndexesPartitionResult.class, this);
}
}
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java
index 1c19856..6bca524 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java
@@ -34,6 +34,7 @@ import java.util.stream.Collectors;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.AtomicConfiguration;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.ConnectorConfiguration;
@@ -53,8 +54,10 @@ import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.spi.encryption.keystore.KeystoreEncryptionSpi;
import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.testframework.junits.WithSystemProperty;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
import static java.lang.String.join;
import static java.lang.System.lineSeparator;
@@ -114,6 +117,27 @@ public abstract class GridCommandHandlerAbstractTest extends GridCommonAbstractT
/** Last operation result. */
protected Object lastOperationResult;
+ /** Persistence flag. */
+ private boolean persistent = true;
+
+ /**
+ * Persistence setter.
+ *
+ * @param pr {@code True} If persistence enable.
+ **/
+ protected void persistenceEnable(boolean pr) {
+ persistent = pr;
+ }
+
+ /**
+ * Persistence getter.
+ *
+ * @return Persistence enable flag.
+ */
+ protected boolean persistenceEnable() {
+ return persistent;
+ }
+
/** {@inheritDoc} */
@Override protected void beforeTestsStarted() throws Exception {
super.beforeTestsStarted();
@@ -192,7 +216,7 @@ public abstract class GridCommandHandlerAbstractTest extends GridCommonAbstractT
.setWalMode(WALMode.LOG_ONLY)
.setCheckpointFrequency(checkpointFreq)
.setDefaultDataRegionConfiguration(
- new DataRegionConfiguration().setMaxSize(50L * 1024 * 1024).setPersistenceEnabled(true)
+ new DataRegionConfiguration().setMaxSize(50L * 1024 * 1024).setPersistenceEnabled(persistent)
);
if (dataRegionConfiguration != null)
@@ -376,17 +400,38 @@ public abstract class GridCommandHandlerAbstractTest extends GridCommonAbstractT
*
* @param ignite Ignite.
* @param countEntries Count of entries.
+ * @param partitions Partitions count.
+ * @param filter Node filter.
*/
- protected void createCacheAndPreload(Ignite ignite, int countEntries) {
+ protected void createCacheAndPreload(
+ Ignite ignite,
+ int countEntries,
+ int partitions,
+ @Nullable IgnitePredicate<ClusterNode> filter
+ ) {
assert nonNull(ignite);
- ignite.createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME)
- .setAffinity(new RendezvousAffinityFunction(false, 32))
- .setBackups(1)
- .setEncryptionEnabled(encriptionEnabled));
+ CacheConfiguration<?, ?> ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+ .setAffinity(new RendezvousAffinityFunction(false, partitions))
+ .setBackups(1);
+
+ if (filter != null)
+ ccfg.setNodeFilter(filter);
+
+ ignite.createCache(ccfg);
IgniteCache<Object, Object> cache = ignite.cache(DEFAULT_CACHE_NAME);
for (int i = 0; i < countEntries; i++)
cache.put(i, i);
}
+
+ /**
+ * Creates default cache and preload some data entries.
+ *
+ * @param ignite Ignite.
+ * @param countEntries Count of entries.
+ */
+ protected void createCacheAndPreload(Ignite ignite, int countEntries) {
+ createCacheAndPreload(ignite, countEntries, 32, null);
+ }
}
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 7442ae0..b60a75a 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -44,6 +44,7 @@ import java.util.stream.IntStream;
import javax.cache.processor.EntryProcessor;
import javax.cache.processor.EntryProcessorException;
import javax.cache.processor.MutableEntry;
+
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteAtomicSequence;
import org.apache.ignite.IgniteCache;
@@ -118,6 +119,7 @@ import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK
import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_UNEXPECTED_ERROR;
import static org.apache.ignite.internal.commandline.CommandList.DEACTIVATE;
import static org.apache.ignite.internal.encryption.AbstractEncryptionTest.MASTER_KEY_NAME_2;
+import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.GRID_NOT_IDLE_MSG;
import static org.apache.ignite.internal.processors.diagnostic.DiagnosticProcessor.DEFAULT_TARGET_FOLDER;
import static org.apache.ignite.testframework.GridTestUtils.assertContains;
import static org.apache.ignite.testframework.GridTestUtils.runAsync;
@@ -1240,7 +1242,7 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb
/** */
@Test
public void testIdleVerifyCheckCrcFailsOnNotIdleCluster() throws Exception {
- checkpointFreq = 100L;
+ checkpointFreq = 1000L;
IgniteEx node = startGrids(2);
@@ -1258,7 +1260,7 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb
ThreadLocalRandom rnd = ThreadLocalRandom.current();
while (!stopFlag.get()) {
- cache.put(rnd.nextInt(), rnd.nextInt());
+ cache.put(rnd.nextInt(1000), rnd.nextInt(1000));
if (Thread.interrupted())
break;
@@ -1275,6 +1277,8 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb
assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--check-crc"));
}
finally {
+ doSleep(checkpointFreq);
+
stopFlag.set(true);
loadThread.join();
@@ -1289,7 +1293,7 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb
String logFile = new String(Files.readAllBytes(new File(logFileName + ".txt").toPath()));
- assertContains(log, logFile, "Checkpoint with dirty pages started! Cluster not idle!");
+ assertContains(log, logFile, GRID_NOT_IDLE_MSG);
}
/**
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
index 40f25cd..6f7e5fc 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
@@ -31,10 +31,10 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
+
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteInterruptedException;
@@ -47,6 +47,7 @@ import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.PartitionUpdateCounter;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.persistence.DbCheckpointListener;
@@ -72,6 +73,7 @@ import org.apache.ignite.internal.util.typedef.T3;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.resources.IgniteInstanceResource;
import org.apache.ignite.resources.LoggerResource;
import org.h2.engine.Session;
@@ -88,7 +90,11 @@ import static java.util.Objects.nonNull;
import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX;
import static org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.GRID_NOT_IDLE_MSG;
+import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.compareUpdateCounters;
+import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.formatUpdateCountersDiff;
import static org.apache.ignite.internal.util.IgniteUtils.error;
+import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.getUpdateCountersSnapshot;
/**
* Closure that locally validates indexes of given caches.
@@ -188,9 +194,11 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
}
/**
+ * Detect groups for further analysis.
*
+ * @return Collection of group id.
*/
- private VisorValidateIndexesJobResult call0() {
+ private Set<Integer> collectGroupIds() {
Set<Integer> grpIds = new HashSet<>();
Set<String> missingCaches = new HashSet<>();
@@ -199,46 +207,49 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
for (String cacheName : cacheNames) {
DynamicCacheDescriptor desc = ignite.context().cache().cacheDescriptor(cacheName);
- if (desc == null) {
+ if (desc == null)
missingCaches.add(cacheName);
-
- continue;
- }
-
- grpIds.add(desc.groupId());
+ else
+ if (ignite.context().cache().cacheGroup(desc.groupId()).affinityNode())
+ grpIds.add(desc.groupId());
}
if (!missingCaches.isEmpty()) {
- StringBuilder strBuilder = new StringBuilder("The following caches do not exist: ");
-
- for (String name : missingCaches)
- strBuilder.append(name).append(", ");
+ String errStr = "The following caches do not exist: " + String.join(", ", missingCaches);
- strBuilder.delete(strBuilder.length() - 2, strBuilder.length());
-
- throw new IgniteException(strBuilder.toString());
+ throw new IgniteException(errStr);
}
}
else {
Collection<CacheGroupContext> groups = ignite.context().cache().cacheGroups();
for (CacheGroupContext grp : groups) {
- if (!grp.systemCache() && !grp.isLocal())
+ if (!grp.systemCache() && !grp.isLocal() && grp.affinityNode())
grpIds.add(grp.groupId());
}
}
- List<Future<Map<PartitionKey, ValidateIndexesPartitionResult>>> procPartFutures = new ArrayList<>();
- List<Future<Map<String, ValidateIndexesPartitionResult>>> procIdxFutures = new ArrayList<>();
- List<T3<CacheGroupContext, GridDhtLocalPartition, Future<CacheSize>>> cacheSizeFutures = new ArrayList<>();
- List<T3<GridCacheContext, Index, Future<T2<Throwable, Long>>>> idxSizeFutures = new ArrayList<>();
+ return grpIds;
+ }
+
+ /**
+ *
+ */
+ private VisorValidateIndexesJobResult call0() {
+ Set<Integer> grpIds = collectGroupIds();
+
+ /** Update counters per partition per group. */
+ final Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrp =
+ getUpdateCountersSnapshot(ignite, grpIds);
+
+ IdleVerifyUtility.IdleChecker idleChecker = new IdleVerifyUtility.IdleChecker(ignite, partsWithCntrsPerGrp);
List<T2<CacheGroupContext, GridDhtLocalPartition>> partArgs = new ArrayList<>();
List<T2<GridCacheContext, Index>> idxArgs = new ArrayList<>();
totalCacheGrps = grpIds.size();
- Map<Integer, IndexIntegrityCheckIssue> integrityCheckResults = integrityCheckIndexesPartitions(grpIds);
+ Map<Integer, IndexIntegrityCheckIssue> integrityCheckResults = integrityCheckIndexesPartitions(grpIds, idleChecker);
GridQueryProcessor qryProcessor = ignite.context().query();
IgniteH2Indexing h2Indexing = (IgniteH2Indexing)qryProcessor.getIndexing();
@@ -283,11 +294,15 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
totalPartitions = partArgs.size();
totalIndexes = idxArgs.size();
- for (T2<CacheGroupContext, GridDhtLocalPartition> t2 : partArgs)
- procPartFutures.add(processPartitionAsync(t2.get1(), t2.get2()));
+ List<Future<Map<PartitionKey, ValidateIndexesPartitionResult>>> procPartFutures = new ArrayList<>(partArgs.size());
+ List<Future<Map<String, ValidateIndexesPartitionResult>>> procIdxFutures = new ArrayList<>(idxArgs.size());
+
+ List<T3<CacheGroupContext, GridDhtLocalPartition, Future<CacheSize>>> cacheSizeFutures = new ArrayList<>(partArgs.size());
+ List<T3<GridCacheContext, Index, Future<T2<Throwable, Long>>>> idxSizeFutures = new ArrayList<>(idxArgs.size());
- for (T2<GridCacheContext, Index> t2 : idxArgs)
- procIdxFutures.add(processIndexAsync(t2.get1(), t2.get2()));
+ partArgs.forEach(k -> procPartFutures.add(processPartitionAsync(k.get1(), k.get2())));
+
+ idxArgs.forEach(k -> procIdxFutures.add(processIndexAsync(k, idleChecker)));
if (checkSizes) {
for (T2<CacheGroupContext, GridDhtLocalPartition> partArg : partArgs) {
@@ -301,7 +316,7 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
GridCacheContext cacheCtx = idxArg.get1();
Index idx = idxArg.get2();
- idxSizeFutures.add(new T3<>(cacheCtx, idx, calcIndexSizeAsync(cacheCtx, idx)));
+ idxSizeFutures.add(new T3<>(cacheCtx, idx, calcIndexSizeAsync(cacheCtx, idx, idleChecker)));
}
}
@@ -332,13 +347,27 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
idxResults.putAll(idxRes);
}
- for (; curCacheSize < cacheSizeFutures.size(); curCacheSize++)
- cacheSizeFutures.get(curCacheSize).get3().get();
+ if (checkSizes) {
+ for (; curCacheSize < cacheSizeFutures.size(); curCacheSize++)
+ cacheSizeFutures.get(curCacheSize).get3().get();
+
+ for (; curIdxSize < idxSizeFutures.size(); curIdxSize++)
+ idxSizeFutures.get(curIdxSize).get3().get();
- for (; curIdxSize < idxSizeFutures.size(); curIdxSize++)
- idxSizeFutures.get(curIdxSize).get3().get();
+ checkSizes(cacheSizeFutures, idxSizeFutures, checkSizeResults);
- checkSizes(cacheSizeFutures, idxSizeFutures, checkSizeResults);
+ Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrpAfterChecks =
+ getUpdateCountersSnapshot(ignite, grpIds);
+
+ List<Integer> diff = compareUpdateCounters(ignite, partsWithCntrsPerGrp, partsWithCntrsPerGrpAfterChecks);
+
+ if (!F.isEmpty(diff)) {
+ String res = formatUpdateCountersDiff(ignite, diff);
+
+ if (!res.isEmpty())
+ throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[" + res + "]");
+ }
+ }
log.warning("ValidateIndexesClosure finished: processed " + totalPartitions + " partitions and "
+ totalIndexes + " indexes.");
@@ -369,8 +398,14 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
/**
* @param grpIds Group ids.
+ * @param idleChecker Idle check closure.
*/
- private Map<Integer, IndexIntegrityCheckIssue> integrityCheckIndexesPartitions(Set<Integer> grpIds) {
+ private Map<Integer, IndexIntegrityCheckIssue> integrityCheckIndexesPartitions(
+ Set<Integer> grpIds,
+ IgniteInClosure<Integer> idleChecker) {
+ if (!checkCrc)
+ return Collections.emptyMap();
+
List<Future<T2<Integer, IndexIntegrityCheckIssue>>> integrityCheckFutures = new ArrayList<>(grpIds.size());
Map<Integer, IndexIntegrityCheckIssue> integrityCheckResults = new HashMap<>();
@@ -382,51 +417,25 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
DbCheckpointListener lsnr = null;
try {
- AtomicBoolean cpFlag = new AtomicBoolean();
-
- if (db instanceof GridCacheDatabaseSharedManager) {
- lsnr = new DbCheckpointListener() {
- @Override public void onMarkCheckpointBegin(Context ctx) {
- /* No-op. */
- }
-
- @Override public void onCheckpointBegin(Context ctx) {
- if (ctx.hasPages())
- cpFlag.set(true);
- }
-
- @Override public void beforeCheckpointBegin(Context ctx) throws IgniteCheckedException {
- /* No-op. */
- }
- };
-
- ((GridCacheDatabaseSharedManager)db).addCheckpointListener(lsnr);
-
- if (IdleVerifyUtility.isCheckpointNow(db))
- throw new GridNotIdleException(IdleVerifyUtility.CLUSTER_NOT_IDLE_MSG);
- }
+ for (Integer grpId: grpIds) {
+ final CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
- if (checkCrc) {
- for (Integer grpId : grpIds) {
- final CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
+ if (grpCtx == null || !grpCtx.persistenceEnabled()) {
+ integrityCheckedIndexes.incrementAndGet();
- if (grpCtx == null || !grpCtx.persistenceEnabled()) {
- integrityCheckedIndexes.incrementAndGet();
-
- continue;
- }
+ continue;
+ }
- Future<T2<Integer, IndexIntegrityCheckIssue>> checkFut =
+ Future<T2<Integer, IndexIntegrityCheckIssue>> checkFut =
calcExecutor.submit(new Callable<T2<Integer, IndexIntegrityCheckIssue>>() {
- @Override public T2<Integer, IndexIntegrityCheckIssue> call() throws Exception {
- IndexIntegrityCheckIssue issue = integrityCheckIndexPartition(grpCtx, cpFlag);
+ @Override public T2<Integer, IndexIntegrityCheckIssue> call() {
+ IndexIntegrityCheckIssue issue = integrityCheckIndexPartition(grpCtx, idleChecker);
return new T2<>(grpCtx.groupId(), issue);
}
});
- integrityCheckFutures.add(checkFut);
- }
+ integrityCheckFutures.add(checkFut);
}
for (Future<T2<Integer, IndexIntegrityCheckIssue>> fut : integrityCheckFutures) {
@@ -452,23 +461,25 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
/**
* @param gctx Cache group context.
- * @param cpFlag Checkpoint status flag.
+ * @param idleChecker Idle check closure.
*/
- private IndexIntegrityCheckIssue integrityCheckIndexPartition(CacheGroupContext gctx, AtomicBoolean cpFlag) {
+ private IndexIntegrityCheckIssue integrityCheckIndexPartition(
+ CacheGroupContext gctx,
+ IgniteInClosure<Integer> idleChecker
+ ) {
GridKernalContext ctx = ignite.context();
GridCacheSharedContext cctx = ctx.cache().context();
try {
FilePageStoreManager pageStoreMgr = (FilePageStoreManager)cctx.pageStore();
- IdleVerifyUtility.checkPartitionsPageCrcSum(pageStoreMgr, gctx, INDEX_PARTITION, FLAG_IDX, cpFlag);
+ IdleVerifyUtility.checkPartitionsPageCrcSum(pageStoreMgr, gctx, INDEX_PARTITION, FLAG_IDX);
+
+ idleChecker.apply(gctx.groupId());
return null;
}
catch (Throwable t) {
- if (cpFlag.get())
- throw new GridNotIdleException("Checkpoint with dirty pages started! Cluster not idle!", t);
-
log.error("Integrity check of index partition of cache group " + gctx.cacheOrGroupName() + " failed", t);
return new IndexIntegrityCheckIssue(gctx.cacheOrGroupName(), t);
@@ -486,11 +497,11 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
* @param part Local partition.
*/
private Future<Map<PartitionKey, ValidateIndexesPartitionResult>> processPartitionAsync(
- final CacheGroupContext grpCtx,
- final GridDhtLocalPartition part
+ CacheGroupContext grpCtx,
+ GridDhtLocalPartition part
) {
return calcExecutor.submit(new Callable<Map<PartitionKey, ValidateIndexesPartitionResult>>() {
- @Override public Map<PartitionKey, ValidateIndexesPartitionResult> call() throws Exception {
+ @Override public Map<PartitionKey, ValidateIndexesPartitionResult> call() {
return processPartition(grpCtx, part);
}
});
@@ -513,17 +524,13 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
if (part.state() != OWNING)
return emptyMap();
- long updateCntrBefore = part.updateCounter();
+ @Nullable PartitionUpdateCounter updCntr = part.dataStore().partUpdateCounter();
- long partSize = part.dataStore().fullSize();
+ PartitionUpdateCounter updateCntrBefore = updCntr == null ? null : updCntr.copy();
GridIterator<CacheDataRow> it = grpCtx.offheap().partitionIterator(part.id());
- Object consId = ignite.context().discovery().localNode().consistentId();
-
- boolean isPrimary = part.primary(grpCtx.topology().readyTopologyVersion());
-
- partRes = new ValidateIndexesPartitionResult(updateCntrBefore, partSize, isPrimary, consId, null);
+ partRes = new ValidateIndexesPartitionResult();
boolean enoughIssues = false;
@@ -630,6 +637,14 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
}
}
}
+
+ PartitionUpdateCounter updateCntrAfter = part.dataStore().partUpdateCounter();
+
+ if (updateCntrAfter != null && !updateCntrAfter.equals(updateCntrBefore)) {
+ throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[grpName=" + grpCtx.cacheOrGroupName() +
+ ", grpId=" + grpCtx.groupId() + ", partId=" + part.id() + "] changed during index validation " +
+ "[before=" + updateCntrBefore + ", after=" + updateCntrAfter + "]");
+ }
}
catch (IgniteCheckedException e) {
error(log, "Failed to process partition [grpId=" + grpCtx.groupId() +
@@ -674,26 +689,34 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
}
/**
- * @param ctx Context.
- * @param idx Index.
+ * @param cacheCtxWithIdx Cache context and appropriate index.
+ * @param idleChecker Idle check closure.
*/
- private Future<Map<String, ValidateIndexesPartitionResult>> processIndexAsync(GridCacheContext ctx, Index idx) {
+ private Future<Map<String, ValidateIndexesPartitionResult>> processIndexAsync(
+ T2<GridCacheContext, Index> cacheCtxWithIdx,
+ IgniteInClosure<Integer> idleChecker
+ ) {
return calcExecutor.submit(new Callable<Map<String, ValidateIndexesPartitionResult>>() {
- @Override public Map<String, ValidateIndexesPartitionResult> call() throws Exception {
- return processIndex(ctx, idx);
+ /** {@inheritDoc} */
+ @Override public Map<String, ValidateIndexesPartitionResult> call() {
+ return processIndex(cacheCtxWithIdx, idleChecker);
}
});
}
/**
- * @param ctx Context.
- * @param idx Index.
+ * @param cacheCtxWithIdx Cache context and appropriate index.
+ * @param idleChecker Idle check closure.
*/
- private Map<String, ValidateIndexesPartitionResult> processIndex(GridCacheContext ctx, Index idx) {
- Object consId = ignite.context().discovery().localNode().consistentId();
+ private Map<String, ValidateIndexesPartitionResult> processIndex(
+ T2<GridCacheContext, Index> cacheCtxWithIdx,
+ IgniteInClosure<Integer> idleChecker
+ ) {
+ GridCacheContext ctx = cacheCtxWithIdx.get1();
- ValidateIndexesPartitionResult idxValidationRes = new ValidateIndexesPartitionResult(
- -1, -1, true, consId, idx.getName());
+ Index idx = cacheCtxWithIdx.get2();
+
+ ValidateIndexesPartitionResult idxValidationRes = new ValidateIndexesPartitionResult();
boolean enoughIssues = false;
@@ -765,7 +788,7 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
}
}
- h2key = (KeyCacheObject)h2Row.key();
+ h2key = h2Row.key();
if (h2Row.link() != 0L) {
CacheDataRow cacheDataStoreRow = ctx.group().offheap().read(ctx, h2key);
@@ -803,6 +826,8 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
idx.getName()
);
+ idleChecker.apply(group.groupId());
+
processedIndexes.incrementAndGet();
printProgressOfIndexValidationIfNeeded();
@@ -839,6 +864,10 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
) {
return calcExecutor.submit(() -> {
try {
+ @Nullable PartitionUpdateCounter updCntr = locPart.dataStore().partUpdateCounter();
+
+ PartitionUpdateCounter updateCntrBefore = updCntr == null ? updCntr : updCntr.copy();
+
int grpId = grpCtx.groupId();
if (failCalcCacheSizeGrpIds.contains(grpId))
@@ -900,6 +929,14 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
.computeIfAbsent(tableName, s -> new AtomicLong()).incrementAndGet();
}
+ PartitionUpdateCounter updateCntrAfter = locPart.dataStore().partUpdateCounter();
+
+ if (updateCntrAfter != null && !updateCntrAfter.equals(updateCntrBefore)) {
+ throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[grpName=" + grpCtx.cacheOrGroupName() +
+ ", grpId=" + grpCtx.groupId() + ", partId=" + locPart.id() + "] changed during size " +
+ "calculation [updCntrBefore=" + updateCntrBefore + ", updCntrAfter=" + updateCntrAfter + "]");
+ }
+
return new CacheSize(null, cacheSizeByTbl);
}
catch (Throwable t) {
@@ -930,11 +967,13 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
*
* @param cacheCtx Cache context.
* @param idx Index.
+ * @param idleChecker Idle check closure.
* @return Future with index size.
*/
private Future<T2<Throwable, Long>> calcIndexSizeAsync(
GridCacheContext cacheCtx,
- Index idx
+ Index idx,
+ IgniteInClosure<Integer> idleChecker
) {
return calcExecutor.submit(() -> {
try {
@@ -947,6 +986,9 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
try {
long indexSize = ignite.context().query().getIndexing().indexSize(cacheName, tblName, idxName);
+
+ idleChecker.apply(cacheCtx.groupId());
+
return new T2<>(null, indexSize);
}
catch (Throwable t) {
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/RebuildIndexTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/RebuildIndexTest.java
index b98bdf2..331e3a9 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/RebuildIndexTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/RebuildIndexTest.java
@@ -34,7 +34,7 @@ import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility;
+import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.internal.visor.verify.ValidateIndexesClosure;
import org.apache.ignite.testframework.ListeningTestLogger;
@@ -176,8 +176,9 @@ public class RebuildIndexTest extends GridCommonAbstractTest {
awaitPartitionMapExchange();
- while (IdleVerifyUtility.isCheckpointNow(node.context().cache().context().database()))
- doSleep(500);
+ forceCheckpoint();
+
+ enableCheckpoints(G.allGrids(), false);
// Validate indexes on start.
ValidateIndexesClosure clo = new ValidateIndexesClosure(Collections.singleton(CACHE_NAME), 0, 0, false, true);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
index 245787b..61ec646 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
@@ -20,19 +20,30 @@ package org.apache.ignite.util;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.cluster.ClusterState;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.lang.IgnitePredicate;
import org.junit.Test;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME;
import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_NAME;
+import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.GRID_NOT_IDLE_MSG;
import static org.apache.ignite.testframework.GridTestUtils.assertContains;
import static org.apache.ignite.testframework.GridTestUtils.assertNotContains;
+import static org.apache.ignite.testframework.GridTestUtils.runAsync;
import static org.apache.ignite.util.GridCommandHandlerIndexingUtils.createAndFillCache;
import static org.apache.ignite.util.GridCommandHandlerIndexingUtils.CACHE_NAME;
import static org.apache.ignite.util.GridCommandHandlerIndexingUtils.GROUP_NAME;
@@ -86,8 +97,142 @@ public class GridCommandHandlerIndexingTest extends GridCommandHandlerClusterPer
String out = testOut.toString();
- assertContains(log, out, "Index validation failed");
- assertContains(log, out, "Checkpoint with dirty pages started! Cluster not idle!");
+ assertContains(log, out, GRID_NOT_IDLE_MSG + "[\"" + GROUP_NAME + "\"]");
+ }
+
+
+ /** Run index validation check on busy cluster. */
+ @Test
+ public void testIdleVerifyCheckFailsOnNotIdleClusterWithOverwriteWithPers() throws Exception {
+ runIdleVerifyCheckCrcFailsOnNotIdleCluster(true);
+ }
+
+ /** Run index validation check on busy cluster. */
+ @Test
+ public void testIdleVerifyCheckFailsOnNotIdleClusterWithOverwriteWithoutPers() throws Exception {
+ persistenceEnable(false);
+
+ runIdleVerifyCheckCrcFailsOnNotIdleCluster(true);
+ }
+
+ /** Run index validation check on busy cluster. */
+ @Test
+ public void testIdleVerifyCheckFailsOnNotIdleClusterWithoutOverwriteWithPers() throws Exception {
+ runIdleVerifyCheckCrcFailsOnNotIdleCluster(false);
+ }
+
+ /** Run index validation check on busy cluster. */
+ @Test
+ public void testIdleVerifyCheckFailsOnNotIdleClusterWithoutOverwriteWithoutPers() throws Exception {
+ persistenceEnable(false);
+
+ runIdleVerifyCheckCrcFailsOnNotIdleCluster(false);
+ }
+
+ /**
+ * Check idle on busy cluster.
+ *
+ * @param allowOverwrite Overwrite param for datastreamer.
+ * @throws Exception
+ */
+ public void runIdleVerifyCheckCrcFailsOnNotIdleCluster(boolean allowOverwrite) throws Exception {
+ IgniteEx ig = startGrids(2);
+
+ ig.cluster().active(true);
+
+ int cntPreload = 100;
+
+ int maxItems = 100000;
+
+ createCacheAndPreload(ig, cntPreload, 1, new CachePredicate(F.asList(ig.name())));
+
+ if (persistenceEnable()) {
+ forceCheckpoint();
+
+ enableCheckpoints(G.allGrids(), false);
+ }
+
+ AtomicBoolean stopFlag = new AtomicBoolean();
+
+ CountDownLatch startLoading = new CountDownLatch(1);
+
+ IgniteInternalFuture f = runAsync(() -> {
+ try (IgniteDataStreamer<Object, Object> ldr = ig.dataStreamer(DEFAULT_CACHE_NAME)) {
+ ldr.allowOverwrite(allowOverwrite);
+
+ ldr.perThreadBufferSize(1);
+
+ boolean addFlag = true;
+
+ int i = cntPreload;
+
+ while (!stopFlag.get()) {
+ if (addFlag)
+ ldr.addData(i, i);
+ else
+ ldr.removeData(i);
+
+ if (i == maxItems / 2)
+ startLoading.countDown();
+
+ if (i % 10 == 0)
+ ldr.flush();
+
+ if (++i == maxItems) {
+ addFlag = !addFlag;
+
+ i = 0;
+ }
+ }
+ }
+ });
+
+ injectTestSystemOut();
+
+ startLoading.await();
+
+ assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", "--check-crc", "--check-sizes"));
+
+ stopFlag.set(true);
+
+ f.get();
+
+ String out = testOut.toString();
+
+ assertContains(log, out, GRID_NOT_IDLE_MSG);
+
+ testOut.reset();
+
+ if (persistenceEnable())
+ enableCheckpoints(G.allGrids(), true);
+
+ assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", "--check-crc", "--check-sizes"));
+
+ out = testOut.toString();
+
+ assertNotContains(log, out, GRID_NOT_IDLE_MSG);
+ }
+
+ /**
+ *
+ */
+ static class CachePredicate implements IgnitePredicate<ClusterNode> {
+ /** */
+ private List<String> excludeNodes;
+
+ /**
+ * @param excludeNodes Nodes names.
+ */
+ public CachePredicate(List<String> excludeNodes) {
+ this.excludeNodes = excludeNodes;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean apply(ClusterNode clusterNode) {
+ String name = clusterNode.attribute(ATTR_IGNITE_INSTANCE_NAME).toString();
+
+ return !excludeNodes.contains(name);
+ }
}
/**
@@ -161,7 +306,7 @@ public class GridCommandHandlerIndexingTest extends GridCommandHandlerClusterPer
private Ignite prepareGridForTest() throws Exception {
Ignite ignite = startGrids(GRID_CNT);
- ignite.cluster().active(true);
+ ignite.cluster().state(ClusterState.ACTIVE);
Ignite client = startGrid(CLIENT_NODE_NAME_PREFIX);