You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/01/31 15:45:54 UTC
[ignite] branch master updated: IGNITE-11030 Fix Pages List to
preserve partition ID consistency. (#5915)
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new ec48853 IGNITE-11030 Fix Pages List to preserve partition ID consistency. (#5915)
ec48853 is described below
commit ec488539933f1cafa0756c7d2bd22b48f0c1e16b
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Thu Jan 31 18:45:47 2019 +0300
IGNITE-11030 Fix Pages List to preserve partition ID consistency. (#5915)
---
.../wal/record/delta/InitNewPageRecord.java | 10 +++
.../cache/persistence/freelist/PagesList.java | 8 ++-
.../cache/persistence/pagemem/PageMemoryImpl.java | 16 ++++-
.../persistence/db/IgniteTcBotInitNewPageTest.java | 81 ++++++++++++++++++++++
.../testsuites/IgnitePdsWithIndexingTestSuite.java | 4 +-
5 files changed, 114 insertions(+), 5 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java
index d0ba2aa..9ed7f35 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -51,6 +52,15 @@ public class InitNewPageRecord extends PageDeltaRecord {
this.ioType = ioType;
this.ioVer = ioVer;
this.newPageId = newPageId;
+
+ int newPartId = PageIdUtils.partId(newPageId);
+ int partId = PageIdUtils.partId(pageId);
+
+ if (newPartId != partId) {
+ throw new AssertionError("Partition consistency failure: " +
+ "newPageId=" + Long.toHexString(newPageId) + " (newPartId: " + newPartId + ") " +
+ "pageId=" + Long.toHexString(pageId) + " (partId: " + partId + ")");
+ }
}
/** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
index 0358975..4359ef4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
@@ -1157,9 +1157,11 @@ public abstract class PagesList extends DataStructure {
decrementBucketSize(bucket);
- if (initIoVers != null)
- dataPageId = initReusedPage(tailId, tailPage, tailAddr, 0, FLAG_DATA, initIoVers.latest());
- else
+ if (initIoVers != null) {
+ int partId = PageIdUtils.partId(tailId);
+
+ dataPageId = initReusedPage(tailId, tailPage, tailAddr, partId, FLAG_DATA, initIoVers.latest());
+ } else
dataPageId = recyclePage(tailId, tailPage, tailAddr, null);
dirty = true;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 88f7ec2..4b236d8 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -1548,7 +1548,7 @@ public class PageMemoryImpl implements PageMemoryEx {
long pageId = PageIO.getPageId(page + PAGE_OVERHEAD);
assert pageId != 0 : U.hexLong(PageHeader.readPageId(page));
- assert PageIO.getVersion(page + PAGE_OVERHEAD) != 0 : U.hexLong(pageId);
+ assert PageIO.getVersion(page + PAGE_OVERHEAD) != 0 : dumpPage(pageId, fullId.groupId());
assert PageIO.getType(page + PAGE_OVERHEAD) != 0 : U.hexLong(pageId);
try {
@@ -1567,6 +1567,20 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/**
+ * Prepares page details for assertion.
+ * @param pageId Page id.
+ * @param grpId Group id.
+ */
+ @NotNull private String dumpPage(long pageId, int grpId) {
+ int pageIdx = PageIdUtils.pageIndex(pageId);
+ int partId = PageIdUtils.partId(pageId);
+ long off = (long)(pageIdx + 1) * pageSize();
+
+ return U.hexLong(pageId) + " (grpId=" + grpId + ", pageIdx=" + pageIdx + ", partId=" + partId + ", offH=" +
+ Long.toHexString(off) + ")";
+ }
+
+ /**
* @param absPtr Absolute pointer to the page.
* @return {@code True} if write lock acquired for the page.
*/
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteTcBotInitNewPageTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteTcBotInitNewPageTest.java
new file mode 100644
index 0000000..8ef517c
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteTcBotInitNewPageTest.java
@@ -0,0 +1,81 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import com.google.common.base.Strings;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+/**
+ * Test creates a lot of index pages in the cache with low number of partitions.<br>
+ * Then cache entries are removed to enforce all pages to come to a free list. <br>
+ * Then creation of data pages with long data will probably result in page rotation.<br>
+ * Expected behaviour: all {@link InitNewPageRecord} should have consistent partition IDs.
+ */
+public class IgniteTcBotInitNewPageTest extends GridCommonAbstractTest {
+ /** Cache name. */
+ public static final String CACHE = "cache";
+
+ /** */
+ @Test
+ public void testInitNewPagePageIdConsistency() throws Exception {
+ IgniteEx ex = startGrid(0);
+
+ IgniteCache<Object, Object> cache = ex.cache(CACHE);
+
+ for (int i = 0; i < 1_000_000; i++)
+ cache.put(i, i);
+
+ cache.clear();
+
+ for (int i = 0; i < 1_000; i++)
+ cache.put(i, Strings.repeat("Apache Ignite", 1000));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ CacheConfiguration<Integer, Object> ccfg = new CacheConfiguration<>(CACHE);
+
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 4));
+
+ cfg.setCacheConfiguration(ccfg);
+
+ DataRegionConfiguration regCfg = new DataRegionConfiguration()
+ .setMaxSize(2L * 1024 * 1024 * 1024)
+ .setPersistenceEnabled(true);
+
+ DataStorageConfiguration dsCfg = new DataStorageConfiguration()
+ .setWalMode(WALMode.LOG_ONLY)
+ .setDefaultDataRegionConfiguration(regCfg);
+
+ cfg.setDataStorageConfiguration(dsCfg);
+
+ return cfg;
+ }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
index 41fb55f..8e14e8c 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
@@ -19,6 +19,7 @@ package org.apache.ignite.testsuites;
import org.apache.ignite.internal.processors.cache.IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest;
import org.apache.ignite.internal.processors.cache.IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.IgniteTcBotInitNewPageTest;
import org.apache.ignite.internal.processors.database.IgniteDbMultiNodeWithIndexingPutGetTest;
import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithIndexingPutGetTest;
import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithIndexingWalRestoreTest;
@@ -38,7 +39,8 @@ import org.junit.runners.Suite;
IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest.class,
IgnitePersistentStoreSchemaLoadTest.class,
IgnitePersistentStoreQueryWithMultipleClassesPerCacheTest.class,
- IgniteTwoRegionsRebuildIndexTest.class
+ IgniteTwoRegionsRebuildIndexTest.class,
+ IgniteTcBotInitNewPageTest.class
})
public class IgnitePdsWithIndexingTestSuite {
}