You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by se...@apache.org on 2021/09/06 12:01:39 UTC
[ignite] branch master updated: IGNITE-15227 Better diagnostic
information for PDS corruption scenarios. Fixes #9292
This is an automated email from the ASF dual-hosted git repository.
sergeychugunov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new eebda74 IGNITE-15227 Better diagnostic information for PDS corruption scenarios. Fixes #9292
eebda74 is described below
commit eebda74d003ad3471a576bc04f151edd70ed1801
Author: Denis Chudov <mo...@gmail.com>
AuthorDate: Mon Sep 6 14:55:56 2021 +0300
IGNITE-15227 Better diagnostic information for PDS corruption scenarios. Fixes #9292
Signed-off-by: Sergey Chugunov <se...@gmail.com>
---
.../AbstractCorruptedPersistenceException.java | 106 +++++++++
...va => CorruptedPartitionMetaPageException.java} | 30 ++-
.../persistence/CorruptedPersistenceException.java | 8 +-
.../cache/persistence/GridCacheOffheapManager.java | 49 ++++-
.../CachePartitionDefragmentationManager.java | 7 +
.../persistence/freelist/AbstractFreeList.java | 34 ++-
.../freelist/CorruptedFreeListException.java | 35 ++-
.../cache/persistence/freelist/PagesList.java | 45 +++-
.../partstorage/PartitionMetaStorage.java | 5 +
.../cache/persistence/tree/BPlusTree.java | 27 ++-
.../persistence/tree/CorruptedTreeException.java | 62 +-----
.../persistence/tree/io/PagePartitionMetaIOV2.java | 2 +-
.../processors/diagnostic/DiagnosticProcessor.java | 18 +-
.../failure/FailureHandlerWithCallback.java} | 22 +-
.../PagesPossibleCorruptionDiagnosticTest.java | 240 +++++++++++++++++++++
.../ignite/testsuites/IgnitePdsTestSuite4.java | 3 +
16 files changed, 592 insertions(+), 101 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/AbstractCorruptedPersistenceException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/AbstractCorruptedPersistenceException.java
new file mode 100644
index 0000000..5156cce
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/AbstractCorruptedPersistenceException.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTreeRuntimeException;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
+import org.jetbrains.annotations.Nullable;
+
+import static java.util.Arrays.asList;
+
+/**
+ * Abstract exception for exceptions related to persistence corruption.
+ */
+public abstract class AbstractCorruptedPersistenceException extends IgniteCheckedException implements CorruptedPersistenceException {
+ /** */
+ protected final T2<Integer, Long>[] pages;
+
+ /**
+ * @param msg Message.
+ * @param cause Cause.
+ * @param pages (groupId, pageId) pairs for pages that might be corrupted.
+ */
+ protected AbstractCorruptedPersistenceException(String msg, @Nullable Throwable cause, T2<Integer, Long>[] pages) {
+ super(msg, cause);
+
+ this.pages = expandPagesArray(pages, cause);
+ }
+
+ /**
+ * @param grpId Group id.
+ * @param pageIds Pages ids.
+ * @return Pairs of (groupId, pageId).
+ */
+ protected static T2<Integer, Long>[] toPagesArray(int grpId, long[] pageIds) {
+ T2<Integer, Long>[] res = (T2<Integer, Long>[])new T2[pageIds.length];
+
+ for (int i = 0; i < pageIds.length; i++)
+ res[i] = new T2<>(grpId, pageIds[i]);
+
+ return res;
+ }
+
+ /**
+ * Add partition meta pages and related pages.
+ * @param pages Pages with group ids.
+ * @param cause Cause exception.
+ * @return Extended list of pages.
+ */
+ protected T2<Integer, Long>[] expandPagesArray(T2<Integer, Long>[] pages, Throwable cause) {
+ Set<T2<Integer, Long>> res = new HashSet<>(asList(pages));
+
+ BPlusTreeRuntimeException treeRuntimeException = X.cause(cause, BPlusTreeRuntimeException.class);
+
+ // Add root exception pages ids if we have.
+ if (treeRuntimeException != null)
+ res.addAll(treeRuntimeException.pages());
+
+ Set<T2<Integer, Long>> partMetaPages = partitionMetaPages(res);
+
+ // Add meta pages for all (group,partition) pairs.
+ res.addAll(partMetaPages);
+
+ return (T2<Integer, Long>[])res.toArray(new T2[0]);
+ }
+
+ /**
+ * @param pages Pages with group ids.
+ * @return Partition meta pages with group ids, for given pages.
+ */
+ protected Set<T2<Integer, Long>> partitionMetaPages(Set<T2<Integer, Long>> pages) {
+ return pages.stream().map(page -> {
+ int grpId = page.get1();
+ int partId = PageIdUtils.partId(page.get2());
+
+ final long partMetaPageId = PageIdUtils.pageId(partId, PageIdAllocator.FLAG_DATA, 0);
+
+ return new T2<>(grpId, partMetaPageId);
+ }).collect(Collectors.toSet());
+ }
+
+ /** {@inheritDoc} */
+ @Override public T2<Integer, Long>[] pages() {
+ return pages;
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CorruptedFreeListException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CorruptedPartitionMetaPageException.java
similarity index 52%
copy from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CorruptedFreeListException.java
copy to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CorruptedPartitionMetaPageException.java
index eb94c63..8a8d4c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CorruptedFreeListException.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CorruptedPartitionMetaPageException.java
@@ -14,25 +14,37 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.ignite.internal.processors.cache.persistence;
-package org.apache.ignite.internal.processors.cache.persistence.freelist;
-
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.persistence.CorruptedPersistenceException;
+import org.apache.ignite.internal.util.typedef.T2;
import org.jetbrains.annotations.Nullable;
/**
- * Exception to distinguish {@link AbstractFreeList} broken invariants.
+ * Exception to distinguish partition meta page broken invariants.
*/
-public class CorruptedFreeListException extends IgniteCheckedException implements CorruptedPersistenceException {
+public class CorruptedPartitionMetaPageException extends AbstractCorruptedPersistenceException {
/** */
private static final long serialVersionUID = 0L;
/**
- * @param msg Message.
+ * @param msg Message.
+ * @param cause Cause.
+ * @param grpId Group id.
+ * @param pages Ids of pages that might be corrupted.
+ */
+ protected CorruptedPartitionMetaPageException(String msg, @Nullable Throwable cause, int grpId, long... pages) {
+ this(msg, cause, toPagesArray(grpId, pages));
+ }
+
+ /**
+ * @param msg Message.
* @param cause Cause.
+ * @param pages (groupId, pageId) pairs for pages that might be corrupted.
*/
- public CorruptedFreeListException(String msg, @Nullable Throwable cause) {
- super(msg, cause);
+ protected CorruptedPartitionMetaPageException(String msg,
+ @Nullable Throwable cause,
+ T2<Integer, Long>[] pages
+ ) {
+ super(msg, cause, pages);
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CorruptedPersistenceException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CorruptedPersistenceException.java
index 6ba7cb3..db84351 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CorruptedPersistenceException.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CorruptedPersistenceException.java
@@ -17,8 +17,14 @@
package org.apache.ignite.internal.processors.cache.persistence;
+import org.apache.ignite.internal.util.typedef.T2;
+
/**
- * Marker interface to distinguish exceptions that were caused by broken persistence datastructures invariants.
+ * Interface to distinguish exceptions that were caused by broken persistence datastructures invariants.
*/
public interface CorruptedPersistenceException {
+ /**
+ * @return (groupId, pageId) pairs for pages that might be corrupted.
+ */
+ public T2<Integer, Long>[] pages();
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index fb7e339..770ceb6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.cache.persistence;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -41,7 +42,6 @@ import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.SystemProperty;
import org.apache.ignite.failure.FailureContext;
-import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.managers.encryption.GridEncryptionManager;
import org.apache.ignite.internal.managers.encryption.ReencryptStateUtils;
import org.apache.ignite.internal.pagemem.FullPageId;
@@ -127,6 +127,7 @@ import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
import org.jetbrains.annotations.Nullable;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
import static org.apache.ignite.internal.processors.cache.GridCacheTtlManager.DFLT_UNWIND_THROTTLING_TIMEOUT;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED;
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
@@ -598,6 +599,12 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
encryptIdx,
encryptCnt
));
+
+ if (changed) {
+ partStore.saveMetadata(grp.statisticsHolderData());
+
+ io.setPartitionMetaStoreReuseListRoot(partMetaPageAddr, partStore.metaPageId());
+ }
}
finally {
pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, changed);
@@ -2198,7 +2205,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
U.error(log, "Unhandled exception during page store initialization. All further operations will " +
"be failed and local node will be stopped.", ex);
- ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, ex));
+ ctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, ex));
throw ex;
}
@@ -2227,6 +2234,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
*/
private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException {
PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
+
IgniteWriteAheadLogManager wal = grp.shared().wal();
int grpId = grp.groupId();
@@ -2300,6 +2308,8 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
pendingTreeAllocated = true;
}
+ checkGapsLinkAndPartMetaStorage(io, partMetaId, pageAddr, grpId, partId);
+
if ((partMetaStoreReuseListRoot = io.getPartitionMetaStoreReuseListRoot(pageAddr)) == 0) {
partMetaStoreReuseListRoot = pageMem.allocatePage(grpId, partId, PageMemory.FLAG_AUX);
@@ -2351,6 +2361,41 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
}
}
+ /**
+ * Checks that links to counter data page and partition meta store are both present or both absent in partition.
+ *
+ * @param io Meta page io.
+ * @param pageAddr Meta page address.
+ * @param grpId Group id.
+ * @param partId Partition id.
+ */
+ private void checkGapsLinkAndPartMetaStorage(PagePartitionMetaIOV3 io, long pageId, long pageAddr, int grpId, int partId) {
+ if (io.getPartitionMetaStoreReuseListRoot(pageAddr) == 0 && io.getGapsLink(pageAddr) != 0) {
+ String msg = "Partition meta page corruption: links to counter data page and partition " +
+ "meta store must both be present, or both be absent in partition [" +
+ "grpId=" + grpId +
+ ", partId=" + partId +
+ ", cntrUpdDataPageId=" + io.getGapsLink(pageAddr) +
+ ", partitionMetaStoreReuseListRoot=" + io.getPartitionMetaStoreReuseListRoot(pageAddr) +
+ ']';
+
+ List<Long> pages = new ArrayList<>();
+
+ pages.add(pageId);
+
+ if (io.getPartitionMetaStoreReuseListRoot(pageAddr) != 0)
+ pages.add(io.getPartitionMetaStoreReuseListRoot(pageAddr));
+
+ if (io.getGapsLink(pageAddr) != 0)
+ pages.add(io.getGapsLink(pageAddr));
+
+ CorruptedPartitionMetaPageException e =
+ new CorruptedPartitionMetaPageException(msg, null, grpId, pages.stream().mapToLong(l -> l).toArray());
+
+ grp.shared().kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
+ }
+ }
+
/** {@inheritDoc} */
@Override public CacheDataTree tree() {
return dataTree;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/defragmentation/CachePartitionDefragmentationManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/defragmentation/CachePartitionDefragmentationManager.java
index 885191f..bedca53 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/defragmentation/CachePartitionDefragmentationManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/defragmentation/CachePartitionDefragmentationManager.java
@@ -808,6 +808,9 @@ public class CachePartitionDefragmentationManager {
long rmvId = oldPartMetaIo.getGlobalRemoveId(oldPartMetaPageAddr);
newPartMetaIo.setGlobalRemoveId(newPartMetaPageAddr, rmvId);
+ long reuseListRoot = oldPartMetaIo.getPartitionMetaStoreReuseListRoot(oldPartMetaPageAddr);
+ newPartMetaIo.setPartitionMetaStoreReuseListRoot(newPartMetaPageAddr, reuseListRoot);
+
// Copy cache sizes for shared cache group.
long oldCountersPageId = oldPartMetaIo.getCountersPageId(oldPartMetaPageAddr);
if (oldCountersPageId != 0L) {
@@ -838,6 +841,10 @@ public class CachePartitionDefragmentationManager {
partCtx.newCacheDataStore.partStorage().insertDataRow(gapsDataRow, IoStatisticsHolderNoOp.INSTANCE);
newPartMetaIo.setGapsLink(newPartMetaPageAddr, gapsDataRow.link());
+
+ newPartMetaIo.setPartitionMetaStoreReuseListRoot(newPartMetaPageAddr,
+ oldPartMetaIo.getPartitionMetaStoreReuseListRoot(oldPartMetaPageAddr)
+ );
}
// Encryption stuff.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java
index b793bbe..df4e75f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java
@@ -590,7 +590,7 @@ public abstract class AbstractFreeList<T extends Storable> extends PagesList imp
throw e;
}
catch (Throwable t) {
- throw new CorruptedFreeListException("Failed to insert data row", t);
+ throw new CorruptedFreeListException("Failed to insert data row", t, grpId);
}
}
@@ -644,7 +644,7 @@ public abstract class AbstractFreeList<T extends Storable> extends PagesList imp
}
}
catch (RuntimeException e) {
- throw new CorruptedFreeListException("Failed to insert data rows", e);
+ throw new CorruptedFreeListException("Failed to insert data rows", e, grpId);
}
}
@@ -784,11 +784,14 @@ public abstract class AbstractFreeList<T extends Storable> extends PagesList imp
return updated;
}
+ catch (AssertionError e) {
+ throw corruptedFreeListException(e);
+ }
catch (IgniteCheckedException | Error e) {
throw e;
}
catch (Throwable t) {
- throw new CorruptedFreeListException("Failed to update data row", t);
+ throw new CorruptedFreeListException("Failed to update data row", t, grpId);
}
}
@@ -807,11 +810,14 @@ public abstract class AbstractFreeList<T extends Storable> extends PagesList imp
return updRes;
}
+ catch (AssertionError e) {
+ throw corruptedFreeListException(e);
+ }
catch (IgniteCheckedException | Error e) {
throw e;
}
catch (Throwable t) {
- throw new CorruptedFreeListException("Failed to update data row", t);
+ throw new CorruptedFreeListException("Failed to update data row", t, grpId);
}
}
@@ -842,11 +848,14 @@ public abstract class AbstractFreeList<T extends Storable> extends PagesList imp
reuseList.addForRecycle(bag);
}
+ catch (AssertionError e) {
+ throw corruptedFreeListException(e);
+ }
catch (IgniteCheckedException | Error e) {
throw e;
}
catch (Throwable t) {
- throw new CorruptedFreeListException("Failed to remove data by link", t);
+ throw new CorruptedFreeListException("Failed to remove data by link", t, grpId);
}
}
@@ -897,11 +906,14 @@ public abstract class AbstractFreeList<T extends Storable> extends PagesList imp
try {
put(bag, 0, 0, 0L, REUSE_BUCKET, IoStatisticsHolderNoOp.INSTANCE);
}
+ catch (AssertionError e) {
+ throw corruptedFreeListException(e);
+ }
catch (IgniteCheckedException | Error e) {
throw e;
}
catch (Throwable t) {
- throw new CorruptedFreeListException("Failed to add page for recycle", t);
+ throw new CorruptedFreeListException("Failed to add page for recycle", t, grpId);
}
}
@@ -912,11 +924,14 @@ public abstract class AbstractFreeList<T extends Storable> extends PagesList imp
try {
return takeEmptyPage(REUSE_BUCKET, null, IoStatisticsHolderNoOp.INSTANCE);
}
+ catch (AssertionError e) {
+ throw corruptedFreeListException(e);
+ }
catch (IgniteCheckedException | Error e) {
throw e;
}
catch (Throwable t) {
- throw new CorruptedFreeListException("Failed to take recycled page", t);
+ throw new CorruptedFreeListException("Failed to take recycled page", t, grpId);
}
}
@@ -932,11 +947,14 @@ public abstract class AbstractFreeList<T extends Storable> extends PagesList imp
try {
return storedPagesCount(REUSE_BUCKET);
}
+ catch (AssertionError e) {
+ throw corruptedFreeListException(e);
+ }
catch (IgniteCheckedException | Error e) {
throw e;
}
catch (Throwable t) {
- throw new CorruptedFreeListException("Failed to count recycled pages", t);
+ throw new CorruptedFreeListException("Failed to count recycled pages", t, grpId);
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CorruptedFreeListException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CorruptedFreeListException.java
index eb94c63..90df5ed 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CorruptedFreeListException.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CorruptedFreeListException.java
@@ -17,22 +17,47 @@
package org.apache.ignite.internal.processors.cache.persistence.freelist;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.persistence.CorruptedPersistenceException;
+import java.util.Collection;
+import org.apache.ignite.internal.processors.cache.persistence.AbstractCorruptedPersistenceException;
+import org.apache.ignite.internal.util.typedef.T2;
import org.jetbrains.annotations.Nullable;
/**
* Exception to distinguish {@link AbstractFreeList} broken invariants.
*/
-public class CorruptedFreeListException extends IgniteCheckedException implements CorruptedPersistenceException {
+public class CorruptedFreeListException extends AbstractCorruptedPersistenceException {
/** */
private static final long serialVersionUID = 0L;
/**
* @param msg Message.
* @param cause Cause.
+ * @param grpId Group id.
+ * @param pageIds Ids of pages that are possibly corrupted.
*/
- public CorruptedFreeListException(String msg, @Nullable Throwable cause) {
- super(msg, cause);
+ public CorruptedFreeListException(String msg, @Nullable Throwable cause, int grpId, Collection<Long> pageIds) {
+ this(msg, cause, grpId, pageIds.stream().mapToLong(Long::longValue).toArray());
+ }
+
+ /**
+ * @param msg Message.
+ * @param cause Cause.
+ * @param grpId Group id.
+ * @param pageIds Ids of pages that are possibly corrupted.
+ */
+ public CorruptedFreeListException(String msg, @Nullable Throwable cause, int grpId, long... pageIds) {
+ this(msg, cause, toPagesArray(grpId, pageIds));
+ }
+
+ /**
+ * @param msg Message.
+ * @param cause Cause.
+ * @param pages (groupId, pageId) pairs for pages that might be corrupted.
+ */
+ public CorruptedFreeListException(String msg,
+ @Nullable Throwable cause,
+ T2<Integer, Long>[] pages
+ ) {
+ super(msg, cause, pages);
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
index 5ccf14f..f96b33b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
@@ -665,7 +665,7 @@ public abstract class PagesList extends DataStructure {
// Tail must exist to be updated.
assert !F.isEmpty(tails) : "Missing tails [bucket=" + bucket + ", tails=" + Arrays.toString(tails) +
- ", metaPage=" + U.hexLong(metaPageId) + ']';
+ ", metaPage=" + U.hexLong(metaPageId) + ", grpId=" + grpId + ']';
idx = findTailIndex(tails, oldTailId, idx);
@@ -1401,9 +1401,8 @@ public abstract class PagesList extends DataStructure {
dirty = true;
- assert !isReuseBucket(bucket) ||
- PageIdUtils.itemId(pageId) > 0 && PageIdUtils.itemId(pageId) <= MAX_ITEMID_NUM
- : "Incorrectly recycled pageId in reuse bucket: " + U.hexLong(pageId);
+ if (isReuseBucket(bucket) && !(PageIdUtils.itemId(pageId) > 0 && PageIdUtils.itemId(pageId) <= MAX_ITEMID_NUM))
+ throw corruptedFreeListException("Incorrectly recycled pageId in reuse bucket: " + U.hexLong(pageId), pageId);
if (isReuseBucket(bucket)) {
byte flag = getFlag(initIoVers);
@@ -1996,6 +1995,44 @@ public abstract class PagesList extends DataStructure {
}
/**
+ * @return Meta page id.
+ */
+ public long metaPageId() {
+ return metaPageId;
+ }
+
+ /**
+ * @param err Error that caused this exception.
+ * @param pageIds Ids of possibly corrupted pages.
+ * @return Exception of type {@link CorruptedFreeListException} that wraps original error and ids of
+ * possibly corrupted pages.
+ */
+ protected CorruptedFreeListException corruptedFreeListException(Throwable err, long... pageIds) {
+ return corruptedFreeListException(err.getMessage(), err, pageIds);
+ }
+
+ /**
+ * @param msg Exception message.
+ * @param pageIds Ids of possibly corrupted pages.
+ * @return Exception of type {@link CorruptedFreeListException} that wraps original error and ids of
+ * possibly corrupted pages.
+ */
+ protected CorruptedFreeListException corruptedFreeListException(String msg, long... pageIds) {
+ return corruptedFreeListException(msg, null, pageIds);
+ }
+
+ /**
+ * @param msg Exception message.
+ * @param err Error that caused this exception.
+ * @param pageIds Ids of possibly corrupted pages.
+ * @return Exception of type {@link CorruptedFreeListException} that wraps original error and ids of
+ * possibly corrupted pages.
+ */
+ protected CorruptedFreeListException corruptedFreeListException(String msg, @Nullable Throwable err, long... pageIds) {
+ return new CorruptedFreeListException(msg, err, grpId, pageIds);
+ }
+
+ /**
* Singleton reuse bag.
*/
private static final class SingletonReuseBag implements ReuseBag {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstorage/PartitionMetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstorage/PartitionMetaStorage.java
index cc0a3b5..6ecbbad 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstorage/PartitionMetaStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstorage/PartitionMetaStorage.java
@@ -48,4 +48,9 @@ public interface PartitionMetaStorage<T extends Storable> {
* Saves storage metadata.
*/
public void saveMetadata(IoStatisticsHolder statHolder) throws IgniteCheckedException;
+
+ /**
+ * @return Meta page id of partition meta storage.
+ */
+ public long metaPageId();
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
index ed777bc..3aed93e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
@@ -52,6 +52,7 @@ import org.apache.ignite.internal.pagemem.wal.record.delta.NewRootInitRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.RemoveRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.ReplaceRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.SplitExistingPageRecord;
+import org.apache.ignite.internal.processors.cache.persistence.AbstractCorruptedPersistenceException;
import org.apache.ignite.internal.processors.cache.persistence.DataStructure;
import org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTrackerManager;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
@@ -1108,6 +1109,9 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements
return cursor;
}
+ catch (AbstractCorruptedPersistenceException e) {
+ throw e;
+ }
catch (IgniteCheckedException e) {
throw new IgniteCheckedException("Runtime failure on bounds: [lower=" + lower + ", upper=" + upper + "]", e);
}
@@ -1141,6 +1145,9 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements
try {
cursor.iterate();
}
+ catch (AbstractCorruptedPersistenceException e) {
+ throw e;
+ }
catch (IgniteCheckedException e) {
throw new IgniteCheckedException("Runtime failure on bounds: [lower=" + lower + ", upper=" + upper + "]", e);
}
@@ -1283,6 +1290,9 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements
}
}
}
+ catch (AbstractCorruptedPersistenceException e) {
+ throw e;
+ }
catch (IgniteCheckedException e) {
throw new IgniteCheckedException("Runtime failure on first row lookup", e);
}
@@ -1326,6 +1336,9 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements
return gLast.find();
}
}
+ catch (AbstractCorruptedPersistenceException e) {
+ throw e;
+ }
catch (IgniteCheckedException e) {
throw new IgniteCheckedException("Runtime failure on last row lookup", e);
}
@@ -1367,6 +1380,9 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements
return (R)g.row;
}
+ catch (AbstractCorruptedPersistenceException e) {
+ throw e;
+ }
catch (IgniteCheckedException e) {
throw new IgniteCheckedException("Runtime failure on lookup row: " + row, e);
}
@@ -1930,7 +1946,7 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements
}
}
}
- catch (UnregisteredClassException | UnregisteredBinaryTypeException e) {
+ catch (UnregisteredClassException | UnregisteredBinaryTypeException | AbstractCorruptedPersistenceException e) {
throw e;
}
catch (IgniteCheckedException e) {
@@ -2091,6 +2107,9 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements
}
}
}
+ catch (AbstractCorruptedPersistenceException e) {
+ throw e;
+ }
catch (IgniteCheckedException e) {
throw new IgniteCheckedException("Runtime failure on search row: " + row, e);
}
@@ -2444,6 +2463,9 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements
}
}
}
+ catch (AbstractCorruptedPersistenceException e) {
+ throw e;
+ }
catch (IgniteCheckedException e) {
throw new IgniteCheckedException("Runtime failure on row: " + row, e);
}
@@ -5642,6 +5664,9 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements
readUnlock(pageId, page, pageAddr);
}
}
+ catch (AbstractCorruptedPersistenceException e) {
+ throw e;
+ }
catch (RuntimeException | AssertionError e) {
throw corruptedTreeException("Runtime failure on cursor iteration", e, grpId, pageId);
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/CorruptedTreeException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/CorruptedTreeException.java
index 108a2cd..fc14099 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/CorruptedTreeException.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/CorruptedTreeException.java
@@ -18,31 +18,19 @@
package org.apache.ignite.internal.processors.cache.persistence.tree;
import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.stream.Collectors;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.pagemem.PageIdAllocator;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.processors.cache.persistence.CorruptedPersistenceException;
+import org.apache.ignite.internal.processors.cache.persistence.AbstractCorruptedPersistenceException;
import org.apache.ignite.internal.util.GridStringBuilder;
import org.apache.ignite.internal.util.typedef.T2;
-import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.jetbrains.annotations.Nullable;
-import static java.util.Arrays.asList;
-
/**
* Exception to distinguish {@link BPlusTree} tree broken invariants.
*/
-public class CorruptedTreeException extends IgniteCheckedException implements CorruptedPersistenceException {
+public class CorruptedTreeException extends AbstractCorruptedPersistenceException {
/** */
private static final long serialVersionUID = 0L;
- /** */
- private final T2<Integer, Long>[] pages;
-
/**
* @param msg Message.
* @param cause Cause.
@@ -91,44 +79,7 @@ public class CorruptedTreeException extends IgniteCheckedException implements Co
@Nullable Throwable cause,
T2<Integer, Long>... pages
) {
- super(getMsg(msg, cacheName, indexName, grpName, pages), cause);
-
- this.pages = expandPagesArray(pages, cause);
- }
-
- /** */
- private static T2<Integer, Long>[] toPagesArray(int grpId, long[] pageIds) {
- T2<Integer, Long>[] res = (T2<Integer, Long>[])new T2[pageIds.length];
-
- for (int i = 0; i < pageIds.length; i++)
- res[i] = new T2<>(grpId, pageIds[i]);
-
- return res;
- }
-
- /** */
- private static T2<Integer, Long>[] expandPagesArray(T2<Integer, Long>[] pages, Throwable cause) {
- Set<T2<Integer, Long>> res = new HashSet<>(asList(pages));
-
- BPlusTreeRuntimeException treeRuntimeException = X.cause(cause, BPlusTreeRuntimeException.class);
-
- // Add root exception pages ids if we have.
- if (treeRuntimeException != null)
- res.addAll(treeRuntimeException.pages());
-
- Set<T2<Integer, Long>> partMetaPages = res.stream().map(page -> {
- int grpId = page.get1();
- int partId = PageIdUtils.partId(page.get2());
-
- final long partMetaPageId = PageIdUtils.pageId(partId, PageIdAllocator.FLAG_DATA, 0);
-
- return new T2<>(grpId, partMetaPageId);
- }).collect(Collectors.toSet());
-
- // Add meta pages for all (group,partition) pairs.
- res.addAll(partMetaPages);
-
- return (T2<Integer, Long>[])res.toArray(new T2[0]);
+ super(getMsg(msg, cacheName, indexName, grpName, pages), cause, pages);
}
/** */
@@ -152,11 +103,4 @@ public class CorruptedTreeException extends IgniteCheckedException implements Co
return stringBuilder.toString();
}
-
- /**
- * @return (groupId, pageId) pairs for pages that might be corrupted.
- */
- public T2<Integer, Long>[] pages() {
- return pages;
- }
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIOV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIOV2.java
index d4eeacb..0d31dd5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIOV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIOV2.java
@@ -31,7 +31,7 @@ public class PagePartitionMetaIOV2 extends PagePartitionMetaIO {
private static final int PENDING_TREE_ROOT_OFF = PagePartitionMetaIO.END_OF_PARTITION_PAGE_META;
/** */
- private static final int PART_META_REUSE_LIST_ROOT_OFF = PENDING_TREE_ROOT_OFF + 8;
+ public static final int PART_META_REUSE_LIST_ROOT_OFF = PENDING_TREE_ROOT_OFF + 8;
/** */
protected static final int GAPS_LINK = PART_META_REUSE_LIST_ROOT_OFF + 8;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/diagnostic/DiagnosticProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/diagnostic/DiagnosticProcessor.java
index ea31ec7..9b810b3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/diagnostic/DiagnosticProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/diagnostic/DiagnosticProcessor.java
@@ -29,9 +29,10 @@ import org.apache.ignite.failure.FailureContext;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
import org.apache.ignite.internal.processors.GridProcessorAdapter;
+import org.apache.ignite.internal.processors.cache.persistence.AbstractCorruptedPersistenceException;
+import org.apache.ignite.internal.processors.cache.persistence.CorruptedPersistenceException;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
-import org.apache.ignite.internal.processors.cache.persistence.tree.CorruptedTreeException;
import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentRouter;
import org.apache.ignite.internal.util.typedef.F;
@@ -95,9 +96,10 @@ public class DiagnosticProcessor extends GridProcessorAdapter {
if (IGNITE_DUMP_PAGE_LOCK_ON_FAILURE)
ctx.cache().context().diagnostic().pageLockTracker().dumpLocksToLog();
- CorruptedTreeException corruptedTreeE = X.cause(failureCtx.error(), CorruptedTreeException.class);
+ CorruptedPersistenceException corruptedPersistenceE =
+ X.cause(failureCtx.error(), AbstractCorruptedPersistenceException.class);
- if (corruptedTreeE != null && !F.isEmpty(corruptedTreeE.pages()) && fileIOFactory != null) {
+ if (corruptedPersistenceE != null && !F.isEmpty(corruptedPersistenceE.pages()) && fileIOFactory != null) {
File[] walDirs = walDirs(ctx);
if (F.isEmpty(walDirs)) {
@@ -109,7 +111,7 @@ public class DiagnosticProcessor extends GridProcessorAdapter {
File corruptedPagesFile = corruptedPagesFile(
diagnosticPath,
fileIOFactory,
- corruptedTreeE.pages()
+ corruptedPersistenceE.pages()
);
String walDirsStr = Arrays.stream(walDirs).map(File::getAbsolutePath)
@@ -123,16 +125,16 @@ public class DiagnosticProcessor extends GridProcessorAdapter {
args += " pages=" + corruptedPagesFile.getAbsolutePath();
- log.warning(corruptedTreeE.getClass().getSimpleName() + " has occurred. " +
+ log.warning(corruptedPersistenceE.getClass().getSimpleName() + " has occurred. " +
"To diagnose it, make a backup of the following directories: " + walDirsStr + ". " +
"Then, run the following command: java -cp <classpath> " +
"org.apache.ignite.development.utils.IgniteWalConverter " + args);
}
catch (Throwable t) {
- String pages = Arrays.stream(corruptedTreeE.pages())
- .map(t2 -> "(" + t2.get1() + ',' + t2.get2() + ')').collect(joining("", "[", "]"));
+ String pages = Arrays.stream(corruptedPersistenceE.pages())
+ .map(t2 -> "" + t2.get1() + ':' + t2.get2()).collect(joining("\n", "", ""));
- log.error("Failed to dump diagnostic info on tree corruption. PageIds=" + pages, t);
+ log.error("Failed to dump diagnostic info of partition corruption. Page ids:\n" + pages, t);
}
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CorruptedPersistenceException.java b/modules/core/src/test/java/org/apache/ignite/failure/FailureHandlerWithCallback.java
similarity index 57%
copy from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CorruptedPersistenceException.java
copy to modules/core/src/test/java/org/apache/ignite/failure/FailureHandlerWithCallback.java
index 6ba7cb3..6e3455b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CorruptedPersistenceException.java
+++ b/modules/core/src/test/java/org/apache/ignite/failure/FailureHandlerWithCallback.java
@@ -14,11 +14,27 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.ignite.failure;
-package org.apache.ignite.internal.processors.cache.persistence;
+import java.util.function.Consumer;
+import org.apache.ignite.Ignite;
/**
- * Marker interface to distinguish exceptions that were caused by broken persistence datastructures invariants.
+ * Failure handler for test purposes that can execute callback to pass failure context inside of it.
*/
-public interface CorruptedPersistenceException {
+public class FailureHandlerWithCallback extends AbstractFailureHandler {
+ /** */
+ private final Consumer<FailureContext> cb;
+
+ /** */
+ public FailureHandlerWithCallback(Consumer<FailureContext> cb) {
+ this.cb = cb;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean handle(Ignite ignite, FailureContext failureCtx) {
+ cb.accept(failureCtx);
+
+ return true;
+ }
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/PagesPossibleCorruptionDiagnosticTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/PagesPossibleCorruptionDiagnosticTest.java
new file mode 100644
index 0000000..d1ab822
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/PagesPossibleCorruptionDiagnosticTest.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterState;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.FailureHandlerWithCallback;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager;
+import org.apache.ignite.internal.processors.cache.PartitionUpdateCounter;
+import org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileVersionCheckingFactory;
+import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.CorruptedFreeListException;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.LongListReuseBag;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseBag;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.testframework.junits.WithSystemProperty;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+import static java.lang.String.format;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
+import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA;
+import static org.apache.ignite.internal.pagemem.PageIdUtils.pageId;
+import static org.apache.ignite.internal.processors.cache.GridCacheUtils.cacheGroupId;
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_DIR_PREFIX;
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_TEMPLATE;
+import static org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIOV2.PART_META_REUSE_LIST_ROOT_OFF;
+
+/**
+ * Tests that grid with corrupted partitions, where meta page is corrupted, fails on start with correct error.
+ */
+@WithSystemProperty(key = IGNITE_PDS_SKIP_CRC, value = "true")
+public class PagesPossibleCorruptionDiagnosticTest extends GridCommonAbstractTest {
+ /** */
+ private static final int PAGE_SIZE = 4096;
+
+ /** */
+ private static final int PAGE_STORE_VER = 2;
+
+ /** */
+ private volatile boolean correctFailure = false;
+
+ /** */
+ private FileVersionCheckingFactory storeFactory = new FileVersionCheckingFactory(
+ new AsyncFileIOFactory(),
+ new AsyncFileIOFactory(),
+ () -> PAGE_SIZE
+ ) {
+ /** {@inheritDoc} */
+ @Override public int latestVersion() {
+ return PAGE_STORE_VER;
+ }
+ };
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ return super.getConfiguration(igniteInstanceName)
+ .setDataStorageConfiguration(new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true))
+ )
+ .setFailureHandler(new FailureHandlerWithCallback(failureCtx ->
+ correctFailure = failureCtx.error() instanceof CorruptedPartitionMetaPageException
+ && ((AbstractCorruptedPersistenceException)failureCtx.error()).pages().length > 0
+ ));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+
+ super.afterTest();
+ }
+
+ /**
+ * @param ignite Ignite instance.
+ * @param partId Partition id.
+ * @return File page store for given partition id.
+ * @throws IgniteCheckedException If failed.
+ */
+ private FilePageStore filePageStore(IgniteEx ignite, int partId) throws IgniteCheckedException {
+ final PdsFolderSettings folderSettings = ignite.context().pdsFolderResolver().resolveFolders();
+
+ File storeWorkDir = new File(folderSettings.persistentStoreRootPath(), folderSettings.folderName());
+
+ File cacheWorkDir = new File(storeWorkDir, CACHE_DIR_PREFIX + DEFAULT_CACHE_NAME);
+
+ File partFile = new File(cacheWorkDir, format(PART_FILE_TEMPLATE, partId));
+
+ return (FilePageStore)storeFactory.createPageStore(FLAG_DATA, partFile, a -> {});
+ }
+
+ /** Tests that node with corrupted partition fails on start. */
+ @Test
+ public void testCorruptedNodeFailsOnStart() throws Exception {
+ IgniteEx ignite = startGrid(0);
+
+ ignite.cluster().state(ClusterState.ACTIVE);
+
+ IgniteCache<Integer, Integer> cache = ignite.getOrCreateCache(new CacheConfiguration<Integer, Integer>(DEFAULT_CACHE_NAME)
+ .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+ .setAffinity(new RendezvousAffinityFunction(false, 1))
+ );
+
+ cache.put(1, 1);
+
+ PartitionUpdateCounter counter = counter(0, DEFAULT_CACHE_NAME, ignite.name());
+
+ counter.update(10, 5);
+
+ forceCheckpoint();
+
+ FilePageStore store = filePageStore(ignite, 0);
+
+ stopAllGrids();
+
+ store.ensure();
+
+ // Index of meta page is always 0.
+ int metaPageIdx = 0;
+
+ long offset = store.headerSize() + metaPageIdx * PAGE_SIZE + PART_META_REUSE_LIST_ROOT_OFF;
+
+ writeLongToFileByOffset(store.getFileAbsolutePath(), offset, 0L);
+
+ try {
+ startGrid(0);
+ }
+ catch (Exception ignored) {
+ /* No-op. */
+ }
+
+ assertTrue(correctFailure);
+ }
+
+ /** Tests page list pages are collected in {@link CorruptedFreeListException}. */
+ @Test
+ @WithSystemProperty(key = IgniteSystemProperties.IGNITE_PAGES_LIST_DISABLE_ONHEAP_CACHING, value = "true")
+ public void testDiagnosticCollectedOnCorruptedPageList() throws Exception {
+ IgniteEx ignite = startGrid(0);
+
+ ignite.cluster().state(ClusterState.ACTIVE);
+
+ IgniteCache<Integer, Integer> cache = ignite.getOrCreateCache(new CacheConfiguration<Integer, Integer>(DEFAULT_CACHE_NAME)
+ .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+ .setAffinity(new RendezvousAffinityFunction(false, 1))
+ );
+
+ cache.put(1, 1);
+
+ cache.remove(1);
+
+ int grpId = cacheGroupId(DEFAULT_CACHE_NAME, null);
+
+ IgniteCacheOffheapManager.CacheDataStore dataStore =
+ ignite.context().cache().cacheGroup(grpId).offheap().cacheDataStores().iterator().next();
+
+ GridCacheOffheapManager.GridCacheDataStore store = (GridCacheOffheapManager.GridCacheDataStore) dataStore;
+
+ AbstractFreeList freeList = store.getCacheStoreFreeList();
+
+ ReuseBag bag = new LongListReuseBag();
+
+ bag.addFreePage(pageId(0, FLAG_DATA, 10));
+ bag.addFreePage(pageId(0, FLAG_DATA, 11));
+
+ T2<Integer, Long>[] pages = null;
+
+ try {
+ freeList.addForRecycle(bag);
+ }
+ catch (CorruptedFreeListException e) {
+ pages = e.pages();
+ }
+
+ assertNotNull(pages);
+ }
+
+ /**
+ * Write any number to any place in the file.
+ *
+ * @param path Path to the file.
+ * @param offset Offset.
+ * @param val Value.
+ * @throws Exception If failed.
+ */
+ private void writeLongToFileByOffset(String path, long offset, long val) throws Exception {
+ ByteBuffer buf = ByteBuffer.allocate(8);
+
+ buf.putLong(val);
+
+ buf.rewind();
+
+ try (FileChannel c = new RandomAccessFile(new File(path), "rw").getChannel()) {
+ c.position(offset);
+
+ c.write(buf);
+ }
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java
index 1a70180..afab955 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java
@@ -47,6 +47,7 @@ import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRestartA
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsSpuriousRebalancingOnNodeJoinTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTaskCancelingTest;
import org.apache.ignite.internal.processors.cache.persistence.NoUnnecessaryRebalanceTest;
+import org.apache.ignite.internal.processors.cache.persistence.PagesPossibleCorruptionDiagnosticTest;
import org.apache.ignite.internal.processors.cache.persistence.PendingTreeCorruptionTest;
import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsCacheWalDisabledOnRebalancingTest;
import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPageReplacementDuringPartitionClearTest;
@@ -146,6 +147,8 @@ public class IgnitePdsTestSuite4 {
GridTestUtils.addTestIfNeeded(suite, PendingTreeCorruptionTest.class, ignoredTests);
+ GridTestUtils.addTestIfNeeded(suite, PagesPossibleCorruptionDiagnosticTest.class, ignoredTests);
+
return suite;
}