You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/01/24 16:46:05 UTC
[ignite] branch ignite-11030 created (now ea3815b)
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a change to branch ignite-11030
in repository https://gitbox.apache.org/repos/asf/ignite.git.
at ea3815b IGNITE-11030 Fix Pages List to preserve partition ID consistency.
This branch includes the following new commits:
new ea3815b IGNITE-11030 Fix Pages List to preserve partition ID consistency.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[ignite] 01/01: IGNITE-11030 Fix Pages List to preserve partition
ID consistency.
Posted by dp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-11030
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit ea3815b0cbf7bb2c0de0ab97f7a2a0d58228221e
Author: Dmitriy Pavlov <dp...@apache.org>
AuthorDate: Thu Jan 24 19:45:55 2019 +0300
IGNITE-11030 Fix Pages List to preserve partition ID consistency.
---
.../wal/record/delta/InitNewPageRecord.java | 10 +++
.../cache/persistence/freelist/PagesList.java | 8 ++-
.../cache/persistence/pagemem/PageMemoryImpl.java | 16 ++++-
.../persistence/db/IgniteTcBotInitNewPageTest.java | 81 ++++++++++++++++++++++
.../testsuites/IgnitePdsWithIndexingTestSuite.java | 4 +-
5 files changed, 114 insertions(+), 5 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java
index d0ba2aa..9ed7f35 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -51,6 +52,15 @@ public class InitNewPageRecord extends PageDeltaRecord {
this.ioType = ioType;
this.ioVer = ioVer;
this.newPageId = newPageId;
+
+ int newPartId = PageIdUtils.partId(newPageId);
+ int partId = PageIdUtils.partId(pageId);
+
+ if (newPartId != partId) {
+ throw new AssertionError("Partition consistency failure: " +
+ "newPageId=" + Long.toHexString(newPageId) + " (newPartId: " + newPartId + ") " +
+ "pageId=" + Long.toHexString(pageId) + " (partId: " + partId + ")");
+ }
}
/** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
index 0358975..4359ef4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
@@ -1157,9 +1157,11 @@ public abstract class PagesList extends DataStructure {
decrementBucketSize(bucket);
- if (initIoVers != null)
- dataPageId = initReusedPage(tailId, tailPage, tailAddr, 0, FLAG_DATA, initIoVers.latest());
- else
+ if (initIoVers != null) {
+ int partId = PageIdUtils.partId(tailId);
+
+ dataPageId = initReusedPage(tailId, tailPage, tailAddr, partId, FLAG_DATA, initIoVers.latest());
+ } else
dataPageId = recyclePage(tailId, tailPage, tailAddr, null);
dirty = true;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 88f7ec2..4b236d8 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -1548,7 +1548,7 @@ public class PageMemoryImpl implements PageMemoryEx {
long pageId = PageIO.getPageId(page + PAGE_OVERHEAD);
assert pageId != 0 : U.hexLong(PageHeader.readPageId(page));
- assert PageIO.getVersion(page + PAGE_OVERHEAD) != 0 : U.hexLong(pageId);
+ assert PageIO.getVersion(page + PAGE_OVERHEAD) != 0 : dumpPage(pageId, fullId.groupId());
assert PageIO.getType(page + PAGE_OVERHEAD) != 0 : U.hexLong(pageId);
try {
@@ -1567,6 +1567,20 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/**
+ * Prepares page details for assertion.
+ * @param pageId Page id.
+ * @param grpId Group id.
+ */
+ @NotNull private String dumpPage(long pageId, int grpId) {
+ int pageIdx = PageIdUtils.pageIndex(pageId);
+ int partId = PageIdUtils.partId(pageId);
+ long off = (long)(pageIdx + 1) * pageSize();
+
+ return U.hexLong(pageId) + " (grpId=" + grpId + ", pageIdx=" + pageIdx + ", partId=" + partId + ", offH=" +
+ Long.toHexString(off) + ")";
+ }
+
+ /**
* @param absPtr Absolute pointer to the page.
* @return {@code True} if write lock acquired for the page.
*/
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteTcBotInitNewPageTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteTcBotInitNewPageTest.java
new file mode 100644
index 0000000..8ef517c
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteTcBotInitNewPageTest.java
@@ -0,0 +1,81 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import com.google.common.base.Strings;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+/**
+ * Test creates a lot of index pages in the cache with low number of partitions.<br>
+ * Then cache entries are removed to enforce all pages to come to a free list. <br>
+ * Then creation of data pages with long data will probably result in page rotation.<br>
+ * Expected behaviour: all {@link InitNewPageRecord} should have consistent partition IDs.
+ */
+public class IgniteTcBotInitNewPageTest extends GridCommonAbstractTest {
+ /** Cache name. */
+ public static final String CACHE = "cache";
+
+ /** */
+ @Test
+ public void testInitNewPagePageIdConsistency() throws Exception {
+ IgniteEx ex = startGrid(0);
+
+ IgniteCache<Object, Object> cache = ex.cache(CACHE);
+
+ for (int i = 0; i < 1_000_000; i++)
+ cache.put(i, i);
+
+ cache.clear();
+
+ for (int i = 0; i < 1_000; i++)
+ cache.put(i, Strings.repeat("Apache Ignite", 1000));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ CacheConfiguration<Integer, Object> ccfg = new CacheConfiguration<>(CACHE);
+
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 4));
+
+ cfg.setCacheConfiguration(ccfg);
+
+ DataRegionConfiguration regCfg = new DataRegionConfiguration()
+ .setMaxSize(2L * 1024 * 1024 * 1024)
+ .setPersistenceEnabled(true);
+
+ DataStorageConfiguration dsCfg = new DataStorageConfiguration()
+ .setWalMode(WALMode.LOG_ONLY)
+ .setDefaultDataRegionConfiguration(regCfg);
+
+ cfg.setDataStorageConfiguration(dsCfg);
+
+ return cfg;
+ }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
index 41fb55f..8e14e8c 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
@@ -19,6 +19,7 @@ package org.apache.ignite.testsuites;
import org.apache.ignite.internal.processors.cache.IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest;
import org.apache.ignite.internal.processors.cache.IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.IgniteTcBotInitNewPageTest;
import org.apache.ignite.internal.processors.database.IgniteDbMultiNodeWithIndexingPutGetTest;
import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithIndexingPutGetTest;
import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithIndexingWalRestoreTest;
@@ -38,7 +39,8 @@ import org.junit.runners.Suite;
IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest.class,
IgnitePersistentStoreSchemaLoadTest.class,
IgnitePersistentStoreQueryWithMultipleClassesPerCacheTest.class,
- IgniteTwoRegionsRebuildIndexTest.class
+ IgniteTwoRegionsRebuildIndexTest.class,
+ IgniteTcBotInitNewPageTest.class
})
public class IgnitePdsWithIndexingTestSuite {
}