You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ib...@apache.org on 2021/04/01 07:09:13 UTC
[ignite] branch master updated: IGNITE-14447 Fixed possible meta
tree corruption after drop index with failed checkpoint scenario. (#8949)
This is an automated email from the ASF dual-hosted git repository.
ibessonov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 120ba5f IGNITE-14447 Fixed possible meta tree corruption after drop index with failed checkpoint scenario. (#8949)
120ba5f is described below
commit 120ba5fd4346d326b443226c1083c74e6eeff15a
Author: ibessonov <be...@gmail.com>
AuthorDate: Thu Apr 1 10:08:42 2021 +0300
IGNITE-14447 Fixed possible meta tree corruption after drop index with failed checkpoint scenario. (#8949)
---
.../cache/IgniteCacheOffheapManager.java | 7 ++
.../cache/IgniteCacheOffheapManagerImpl.java | 5 +
.../cache/persistence/GridCacheOffheapManager.java | 5 +
.../processors/cache/persistence/IndexStorage.java | 12 ++
.../cache/persistence/IndexStorageImpl.java | 13 +++
.../h2/DurableBackgroundCleanupIndexTreeTask.java | 95 ++++++++++++++-
.../processors/query/h2/database/H2TreeIndex.java | 4 +-
.../CleanupIndexTreeCheckpointFailoverTest.java | 127 +++++++++++++++++++++
...teCacheWithIndexingAndPersistenceTestSuite.java | 4 +-
9 files changed, 265 insertions(+), 7 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
index 61e1605..c839574 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
@@ -560,6 +560,13 @@ public interface IgniteCacheOffheapManager {
* @param idxName Index name.
* @throws IgniteCheckedException If failed.
*/
+ public @Nullable RootPage findRootPageForIndex(int cacheId, String idxName, int segment) throws IgniteCheckedException;
+
+ /**
+ * @param cacheId Cache ID.
+ * @param idxName Index name.
+ * @throws IgniteCheckedException If failed.
+ */
public void dropRootPageForIndex(int cacheId, String idxName, int segment) throws IgniteCheckedException;
/**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index 32d59aa..71aca1d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -1118,6 +1118,11 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
}
/** {@inheritDoc} */
+ @Override public @Nullable RootPage findRootPageForIndex(int cacheId, String idxName, int segment) throws IgniteCheckedException {
+ return null; // No-op.
+ }
+
+ /** {@inheritDoc} */
@Override public void dropRootPageForIndex(int cacheId, String idxName, int segment) throws IgniteCheckedException {
// No-op.
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index af0c4d7..4f1b61c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -1063,6 +1063,11 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
}
/** {@inheritDoc} */
+ @Override public @Nullable RootPage findRootPageForIndex(int cacheId, String idxName, int segment) throws IgniteCheckedException {
+ return indexStorage.findCacheIndex(cacheId, idxName, segment);
+ }
+
+ /** {@inheritDoc} */
@Override public void dropRootPageForIndex(int cacheId, String idxName, int segment) throws IgniteCheckedException {
indexStorage.dropCacheIndex(cacheId, idxName, segment);
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorage.java
index f9c6031..9366d70 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorage.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.cache.persistence;
import java.util.Collection;
import org.apache.ignite.IgniteCheckedException;
+import org.jetbrains.annotations.Nullable;
/**
* Meta store.
@@ -46,6 +47,17 @@ public interface IndexStorage {
public RootPage allocateIndex(String idxName) throws IgniteCheckedException;
/**
+ * Find index root.
+ *
+ * @param cacheId Cache ID.
+ * @param idxName Index name.
+ * @param segment Segment.
+ * @return Root ID or null if no page was found.
+ * @throws IgniteCheckedException If failed.
+ */
+ public @Nullable RootPage findCacheIndex(Integer cacheId, String idxName, int segment) throws IgniteCheckedException;
+
+ /**
* Deallocate index page and remove from tree.
*
* @param cacheId Cache ID.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java
index 94f7feb..00bbae6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java
@@ -164,6 +164,19 @@ public class IndexStorageImpl implements IndexStorage {
}
/** {@inheritDoc} */
+ @Override public @Nullable RootPage findCacheIndex(Integer cacheId, String idxName, int segment)
+ throws IgniteCheckedException
+ {
+ idxName = maskCacheIndexName(cacheId, idxName, segment);
+
+ byte[] idxNameBytes = idxName.getBytes(StandardCharsets.UTF_8);
+
+ final IndexItem row = metaTree.findOne(new IndexItem(idxNameBytes, 0));
+
+ return row != null ? new RootPage(new FullPageId(row.pageId, grpId), false) : null;
+ }
+
+ /** {@inheritDoc} */
@Override public RootPage dropCacheIndex(Integer cacheId, String idxName, int segment)
throws IgniteCheckedException {
String maskedIdxName = maskCacheIndexName(cacheId, idxName, segment);
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DurableBackgroundCleanupIndexTreeTask.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DurableBackgroundCleanupIndexTreeTask.java
index 4d916605..5305c17 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DurableBackgroundCleanupIndexTreeTask.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DurableBackgroundCleanupIndexTreeTask.java
@@ -25,7 +25,12 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.metric.IoStatisticsHolderIndex;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager;
+import org.apache.ignite.internal.processors.cache.persistence.RootPage;
import org.apache.ignite.internal.processors.cache.persistence.metastorage.pendingtask.DurableBackgroundTask;
import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIoResolver;
@@ -61,6 +66,9 @@ public class DurableBackgroundCleanupIndexTreeTask implements DurableBackgroundT
private String schemaName;
/** */
+ private final String treeName;
+
+ /** */
private String idxName;
/** */
@@ -73,6 +81,7 @@ public class DurableBackgroundCleanupIndexTreeTask implements DurableBackgroundT
String cacheGrpName,
String cacheName,
String schemaName,
+ String treeName,
String idxName
) {
this.rootPages = rootPages;
@@ -81,6 +90,7 @@ public class DurableBackgroundCleanupIndexTreeTask implements DurableBackgroundT
this.cacheGrpName = cacheGrpName;
this.cacheName = cacheName;
this.schemaName = schemaName;
+ this.treeName = treeName;
this.idxName = idxName;
this.id = UUID.randomUUID().toString();
}
@@ -99,6 +109,39 @@ public class DurableBackgroundCleanupIndexTreeTask implements DurableBackgroundT
GridCacheContext cctx = ctx.cache().context().cacheContext(CU.cacheId(cacheName));
+ int grpId = CU.cacheGroupId(cacheName, cacheGrpName);
+
+ CacheGroupContext grpCtx = ctx.cache().cacheGroup(grpId);
+
+ // If group context is null, it means that group doesn't exist and we don't need this task anymore.
+ if (grpCtx == null)
+ return;
+
+ IgniteCacheOffheapManager offheap = grpCtx.offheap();
+
+ if (treeName != null) {
+ ctx.cache().context().database().checkpointReadLock();
+
+ try {
+ int cacheId = CU.cacheId(cacheName);
+
+ for (int segment = 0; segment < rootPages.size(); segment++) {
+ try {
+ RootPage rootPage = offheap.findRootPageForIndex(cacheId, treeName, segment);
+
+ if (rootPage != null && rootPages.get(segment) == rootPage.pageId().pageId())
+ offheap.dropRootPageForIndex(cacheId, treeName, segment);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+ }
+ finally {
+ ctx.cache().context().database().checkpointReadUnlock();
+ }
+ }
+
IoStatisticsHolderIndex stats = new IoStatisticsHolderIndex(
SORTED_INDEX,
cctx.name(),
@@ -106,11 +149,25 @@ public class DurableBackgroundCleanupIndexTreeTask implements DurableBackgroundT
cctx.kernalContext().metric()
);
+ PageMemory pageMem = grpCtx.dataRegion().pageMemory();
+
for (int i = 0; i < rootPages.size(); i++) {
Long rootPage = rootPages.get(i);
assert rootPage != null;
+ if (skipDeletedRoot(grpId, pageMem, rootPage)) {
+ ctx.log(getClass()).warning(S.toString("Skipping deletion of the index tree",
+ "cacheGrpName", cacheGrpName, false,
+ "cacheName", cacheName, false,
+ "idxName", idxName, false,
+ "segment", i, false,
+ "rootPageId", PageIdUtils.toDetailString(rootPage), false
+ ));
+
+ continue;
+ }
+
// Below we create a fake index tree using it's root page, stubbing some parameters,
// because we just going to free memory pages that are occupied by tree structure.
try {
@@ -123,12 +180,12 @@ public class DurableBackgroundCleanupIndexTreeTask implements DurableBackgroundT
idxName,
cacheName,
null,
- cctx.offheap().reuseListForIndex(treeName),
- CU.cacheGroupId(cacheName, cacheGrpName),
+ offheap.reuseListForIndex(treeName),
+ grpId,
cacheGrpName,
- cctx.dataRegion().pageMemory(),
+ pageMem,
ctx.cache().context().wal(),
- cctx.offheap().globalRemoveId(),
+ offheap.globalRemoveId(),
rootPage,
false,
Collections.emptyList(),
@@ -173,6 +230,36 @@ public class DurableBackgroundCleanupIndexTreeTask implements DurableBackgroundT
}
}
+ /**
+ * Checks that pageId is still relevant and has not been deleted / reused.
+ * @param grpId Cache group id.
+ * @param pageMem Page memory instance.
+ * @param rootPageId Root page identifier.
+ * @return {@code true} if root page was deleted/reused, {@code false} otherwise.
+ */
+ private boolean skipDeletedRoot(int grpId, PageMemory pageMem, long rootPageId) {
+ try {
+ long page = pageMem.acquirePage(grpId, rootPageId);
+
+ try {
+ long pageAddr = pageMem.readLock(grpId, rootPageId, page);
+
+ try {
+ return pageAddr == 0;
+ }
+ finally {
+ pageMem.readUnlock(grpId, rootPageId, page);
+ }
+ }
+ finally {
+ pageMem.releasePage(grpId, rootPageId, page);
+ }
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException("Cannot acquire tree root page.", e);
+ }
+ }
+
/** {@inheritDoc} */
@Override public void complete() {
completed = true;
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java
index 237f191..821af4f 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java
@@ -26,7 +26,6 @@ import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
-
import javax.cache.CacheException;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
@@ -598,9 +597,10 @@ public class H2TreeIndex extends H2TreeIndexBase {
DurableBackgroundTask task = new DurableBackgroundCleanupIndexTreeTask(
rootPages,
trees,
- cctx.group().name(),
+ cctx.group().name() == null ? cctx.cache().name() : cctx.group().name(),
cctx.cache().name(),
table.getSchema().getName(),
+ treeName,
idxName
);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/CleanupIndexTreeCheckpointFailoverTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/CleanupIndexTreeCheckpointFailoverTest.java
new file mode 100644
index 0000000..6aa5629
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/CleanupIndexTreeCheckpointFailoverTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.cluster.ClusterState;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class CleanupIndexTreeCheckpointFailoverTest extends GridCommonAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ DataStorageConfiguration dsCfg = new DataStorageConfiguration();
+ dsCfg.setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true));
+
+ cfg.setDataStorageConfiguration(dsCfg);
+
+ return cfg;
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ @Test
+ public void testCorruptedTree() throws Exception {
+ cleanPersistenceDir();
+
+ IgniteEx ig = startGrid(0);
+ ig.cluster().state(ClusterState.ACTIVE);
+
+ CacheConfiguration<Key, Value> cfg = new CacheConfiguration<Key, Value>()
+ .setIndexedTypes(Key.class, Value.class).setName("test");
+
+ IgniteCache<Key, Value> cache = ig.getOrCreateCache(cfg);
+
+ cache.query(new SqlFieldsQuery("create index myindex on value (a asc)")).getAll();
+
+ for (int i = 0; i < 5000; i++)
+ cache.put(new Key(i), new Value(String.valueOf(i), "b" + i));
+
+ ig.context().cache().context().database().wakeupForCheckpoint("test").get();
+
+ cache.query(new SqlFieldsQuery("drop index myindex")).getAll();
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)ig.context().cache().context()
+ .database();
+
+ U.sleep(1000);
+
+ dbMgr.enableCheckpoints(false);
+
+ stopGrid(0, true);
+
+ ig = startGrid(0);
+
+ cache = ig.cache("test");
+
+ for (int i = 0; i < 5000; i += 2)
+ cache.remove(new Key(i));
+
+ cache.query(new SqlFieldsQuery("create index myindex on value (a asc)")).getAll();
+
+ for (int i = 0; i < 5000; i++)
+ cache.put(new Key(i), new Value(String.valueOf(i), "b" + i));
+ }
+
+ /**
+ */
+ private static class Key {
+ /** */
+ int id;
+
+ /**
+ */
+ Key(int id) {
+ this.id = id;
+ }
+ }
+
+ /**
+ */
+ private static class Value {
+ /** */
+ @QuerySqlField
+ String a;
+
+ /** */
+ @QuerySqlField
+ String b;
+
+ /**
+ */
+ Value(String a, String b) {
+ this.a = a;
+ this.b = b;
+ }
+ }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java
index 175e930..060759d8 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java
@@ -19,6 +19,7 @@ package org.apache.ignite.testsuites;
import org.apache.ignite.internal.processors.cache.StartCachesInParallelTest;
import org.apache.ignite.internal.processors.cache.index.IoStatisticsBasicIndexSelfTest;
+import org.apache.ignite.internal.processors.query.CleanupIndexTreeCheckpointFailoverTest;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
@@ -28,7 +29,8 @@ import org.junit.runners.Suite;
@RunWith(Suite.class)
@Suite.SuiteClasses({
StartCachesInParallelTest.class,
- IoStatisticsBasicIndexSelfTest.class
+ IoStatisticsBasicIndexSelfTest.class,
+ CleanupIndexTreeCheckpointFailoverTest.class
})
public class IgniteCacheWithIndexingAndPersistenceTestSuite {
}