You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by tk...@apache.org on 2025/08/21 16:23:34 UTC
(ignite-3) 03/03: IGNITE-26233 start introduce DirtyFullPageId
This is an automated email from the ASF dual-hosted git repository.
tkalkirill pushed a commit to branch ignite-26233-2
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit 9add37d481380f1df7cb74332a255572c89e3777
Author: Kirill Tkalenko <tk...@yandex.ru>
AuthorDate: Thu Aug 21 19:23:12 2025 +0300
IGNITE-26233 start introduce DirtyFullPageId
---
.../pagememory/persistence/DirtyFullPageId.java | 10 ++++++++
.../persistence/PersistentPageMemory.java | 27 ++++++++++++++--------
.../PersistentPageMemoryMvTableStorageTest.java | 6 +++++
.../pagememory/PersistentPageMemoryNoLoadTest.java | 24 +++++++++++--------
4 files changed, 49 insertions(+), 18 deletions(-)
diff --git a/modules/page-memory/src/main/java/org/apache/ignite/internal/pagememory/persistence/DirtyFullPageId.java b/modules/page-memory/src/main/java/org/apache/ignite/internal/pagememory/persistence/DirtyFullPageId.java
index 591a1d6af07..c3e18d7b7de 100644
--- a/modules/page-memory/src/main/java/org/apache/ignite/internal/pagememory/persistence/DirtyFullPageId.java
+++ b/modules/page-memory/src/main/java/org/apache/ignite/internal/pagememory/persistence/DirtyFullPageId.java
@@ -37,6 +37,16 @@ public final class DirtyFullPageId extends FullPageId {
this.partitionGeneration = partitionGeneration;
}
+ /**
+ * Constructor.
+ *
+ * @param fullPageId Full page ID.
+ * @param partitionGeneration Partition generation.
+ */
+ public DirtyFullPageId(FullPageId fullPageId, int partitionGeneration) {
+ this(fullPageId.pageId(), fullPageId.groupId(), partitionGeneration);
+ }
+
/** Returns partition generation. */
public int partitionGeneration() {
return partitionGeneration;
diff --git a/modules/page-memory/src/main/java/org/apache/ignite/internal/pagememory/persistence/PersistentPageMemory.java b/modules/page-memory/src/main/java/org/apache/ignite/internal/pagememory/persistence/PersistentPageMemory.java
index 0d271a60189..7dba8dd2784 100644
--- a/modules/page-memory/src/main/java/org/apache/ignite/internal/pagememory/persistence/PersistentPageMemory.java
+++ b/modules/page-memory/src/main/java/org/apache/ignite/internal/pagememory/persistence/PersistentPageMemory.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.pagememory.persistence;
import static java.lang.System.lineSeparator;
+import static java.util.stream.Collectors.toSet;
import static org.apache.ignite.internal.pagememory.FullPageId.NULL_PAGE;
import static org.apache.ignite.internal.pagememory.io.PageIo.getCrc;
import static org.apache.ignite.internal.pagememory.io.PageIo.getPageId;
@@ -29,6 +30,7 @@ import static org.apache.ignite.internal.pagememory.persistence.CheckpointUrgenc
import static org.apache.ignite.internal.pagememory.persistence.CheckpointUrgency.SHOULD_TRIGGER;
import static org.apache.ignite.internal.pagememory.persistence.PageHeader.PAGE_LOCK_OFFSET;
import static org.apache.ignite.internal.pagememory.persistence.PageHeader.PAGE_OVERHEAD;
+import static org.apache.ignite.internal.pagememory.persistence.PageHeader.UNKNOWN_PARTITION_GENERATION;
import static org.apache.ignite.internal.pagememory.persistence.PageHeader.dirty;
import static org.apache.ignite.internal.pagememory.persistence.PageHeader.fullPageId;
import static org.apache.ignite.internal.pagememory.persistence.PageHeader.isAcquired;
@@ -1273,16 +1275,20 @@ public class PersistentPageMemory implements PageMemory {
* or not.
*/
private void setDirty(FullPageId pageId, long absPtr, boolean dirty, boolean forceAdd) {
- boolean wasDirty = dirty(absPtr, dirty);
+ int partGen = partitionGeneration(absPtr);
+
+ assert partGen != UNKNOWN_PARTITION_GENERATION : pageId;
if (dirty) {
- assert checkpointTimeoutLock.checkpointLockIsHeldByThread();
- assert pageIndex(pageId.pageId()) != 0 : "Partition meta should only be updated via the instance of PartitionMeta.";
+ assert checkpointTimeoutLock.checkpointLockIsHeldByThread() : pageId;
+ assert pageIndex(pageId.pageId()) != 0 : "Partition meta should only be updated via the instance of PartitionMeta: " + pageId;
+
+ boolean wasDirty = dirty(absPtr, dirty);
if (!wasDirty || forceAdd) {
Segment seg = segment(pageId.groupId(), pageId.pageId());
- if (seg.dirtyPages.add(pageId)) {
+ if (seg.dirtyPages.add(new DirtyFullPageId(pageId, partGen))) {
long dirtyPagesCnt = seg.dirtyPagesCntr.incrementAndGet();
if (dirtyPagesCnt >= seg.dirtyPagesSoftThreshold) {
@@ -1301,7 +1307,7 @@ public class PersistentPageMemory implements PageMemory {
} else {
Segment seg = segment(pageId.groupId(), pageId.pageId());
- if (seg.dirtyPages.remove(pageId)) {
+ if (seg.dirtyPages.remove(new DirtyFullPageId(pageId, partGen))) {
seg.dirtyPagesCntr.decrementAndGet();
}
}
@@ -1333,13 +1339,13 @@ public class PersistentPageMemory implements PageMemory {
* Returns a collection of all pages currently marked as dirty. Will create a collection copy.
*/
@TestOnly
- public Set<FullPageId> dirtyPages() {
+ public Set<DirtyFullPageId> dirtyPages() {
Segment[] segments = this.segments;
if (segments == null) {
return Set.of();
}
- Set<FullPageId> res = new HashSet<>((int) loadedPages());
+ var res = new HashSet<DirtyFullPageId>();
for (Segment seg : segments) {
res.addAll(seg.dirtyPages);
@@ -1398,7 +1404,7 @@ public class PersistentPageMemory implements PageMemory {
private long memPerRepl;
/** Pages marked as dirty since the last checkpoint. */
- private volatile Set<FullPageId> dirtyPages = ConcurrentHashMap.newKeySet();
+ private volatile Set<DirtyFullPageId> dirtyPages = ConcurrentHashMap.newKeySet();
/** Atomic size counter for {@link #dirtyPages}. */
private final AtomicLong dirtyPagesCntr = new AtomicLong();
@@ -2185,7 +2191,10 @@ public class PersistentPageMemory implements PageMemory {
dataRegionConfiguration.name(), i
);
- Set<FullPageId> segmentDirtyPages = segment.dirtyPages;
+ // TODO: IGNITE-26233 Вот конечно исправить надо, пока тупо для удобства
+ Set<FullPageId> segmentDirtyPages = segment.dirtyPages.stream()
+ .map(p -> new FullPageId(p.pageId(), p.groupId()))
+ .collect(toSet());
dirtyPageIds[i] = segmentDirtyPages;
segment.checkpointPages = new CheckpointPages(segmentDirtyPages, checkpointProgress);
diff --git a/modules/storage-page-memory/src/test/java/org/apache/ignite/internal/storage/pagememory/PersistentPageMemoryMvTableStorageTest.java b/modules/storage-page-memory/src/test/java/org/apache/ignite/internal/storage/pagememory/PersistentPageMemoryMvTableStorageTest.java
index ed78919baca..9127ff378eb 100644
--- a/modules/storage-page-memory/src/test/java/org/apache/ignite/internal/storage/pagememory/PersistentPageMemoryMvTableStorageTest.java
+++ b/modules/storage-page-memory/src/test/java/org/apache/ignite/internal/storage/pagememory/PersistentPageMemoryMvTableStorageTest.java
@@ -64,6 +64,7 @@ import org.apache.ignite.internal.util.Constants;
import org.apache.ignite.internal.util.IgniteUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
@@ -272,6 +273,7 @@ public class PersistentPageMemoryMvTableStorageTest extends AbstractMvTableStora
});
}
+ @Disabled("https://issues.apache.org/jira/browse/IGNITE-26233")
@Test
void createMvPartitionStorageAndDoCheckpointInParallel() {
for (int i = 0; i < 10; i++) {
@@ -298,6 +300,7 @@ public class PersistentPageMemoryMvTableStorageTest extends AbstractMvTableStora
}
}
+ @Disabled("https://issues.apache.org/jira/browse/IGNITE-26233")
@Test
void destroyMvPartitionStorageAndDoCheckpointInParallel() {
for (int i = 0; i < 10; i++) {
@@ -310,6 +313,7 @@ public class PersistentPageMemoryMvTableStorageTest extends AbstractMvTableStora
}
}
+ @Disabled("https://issues.apache.org/jira/browse/IGNITE-26233")
@Test
void startRebalancePartitionAndDoCheckpointInParallel() {
getOrCreateMvPartition(PARTITION_ID);
@@ -324,6 +328,7 @@ public class PersistentPageMemoryMvTableStorageTest extends AbstractMvTableStora
}
}
+ @Disabled("https://issues.apache.org/jira/browse/IGNITE-26233")
@Test
void abortRebalancePartitionAndDoCheckpointInParallel() {
getOrCreateMvPartition(PARTITION_ID);
@@ -338,6 +343,7 @@ public class PersistentPageMemoryMvTableStorageTest extends AbstractMvTableStora
}
}
+ @Disabled("https://issues.apache.org/jira/browse/IGNITE-26233")
@Test
void finishRebalancePartitionAndDoCheckpointInParallel() {
getOrCreateMvPartition(PARTITION_ID);
diff --git a/modules/storage-page-memory/src/test/java/org/apache/ignite/internal/storage/pagememory/PersistentPageMemoryNoLoadTest.java b/modules/storage-page-memory/src/test/java/org/apache/ignite/internal/storage/pagememory/PersistentPageMemoryNoLoadTest.java
index e6e53d4c4db..4214bc27c48 100644
--- a/modules/storage-page-memory/src/test/java/org/apache/ignite/internal/storage/pagememory/PersistentPageMemoryNoLoadTest.java
+++ b/modules/storage-page-memory/src/test/java/org/apache/ignite/internal/storage/pagememory/PersistentPageMemoryNoLoadTest.java
@@ -22,10 +22,12 @@ import static org.apache.ignite.internal.pagememory.persistence.CheckpointUrgenc
import static org.apache.ignite.internal.pagememory.persistence.CheckpointUrgency.NOT_REQUIRED;
import static org.apache.ignite.internal.pagememory.persistence.CheckpointUrgency.SHOULD_TRIGGER;
import static org.apache.ignite.internal.pagememory.persistence.PageHeader.PAGE_OVERHEAD;
+import static org.apache.ignite.internal.pagememory.persistence.PageHeader.partitionGeneration;
import static org.apache.ignite.internal.pagememory.persistence.checkpoint.CheckpointState.FINISHED;
import static org.apache.ignite.internal.pagememory.persistence.checkpoint.CheckpointState.PAGES_SORTED;
import static org.apache.ignite.internal.pagememory.persistence.checkpoint.CheckpointTestUtils.mockCheckpointTimeoutLock;
import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully;
+import static org.apache.ignite.internal.util.CollectionUtils.union;
import static org.apache.ignite.internal.util.Constants.MiB;
import static org.apache.ignite.internal.util.GridUnsafe.allocateBuffer;
import static org.apache.ignite.internal.util.GridUnsafe.freeBuffer;
@@ -64,8 +66,8 @@ import org.apache.ignite.internal.pagememory.PageMemory;
import org.apache.ignite.internal.pagememory.configuration.CheckpointConfiguration;
import org.apache.ignite.internal.pagememory.configuration.PersistentDataRegionConfiguration;
import org.apache.ignite.internal.pagememory.io.PageIoRegistry;
+import org.apache.ignite.internal.pagememory.persistence.DirtyFullPageId;
import org.apache.ignite.internal.pagememory.persistence.GroupPartitionId;
-import org.apache.ignite.internal.pagememory.persistence.PageHeader;
import org.apache.ignite.internal.pagememory.persistence.PartitionMeta.PartitionMetaSnapshot;
import org.apache.ignite.internal.pagememory.persistence.PartitionMetaManager;
import org.apache.ignite.internal.pagememory.persistence.PersistentPageMemory;
@@ -175,9 +177,13 @@ public class PersistentPageMemoryNoLoadTest extends AbstractPageMemoryNoLoadSelf
checkpointManager.checkpointTimeoutLock().checkpointReadLock();
try {
- Set<FullPageId> dirtyPages = Set.of(createDirtyPage(pageMemory), createDirtyPage(pageMemory));
-
+ Set<DirtyFullPageId> dirtyPages = Set.of(createDirtyPage(pageMemory), createDirtyPage(pageMemory));
assertThat(pageMemory.dirtyPages(), equalTo(dirtyPages));
+
+ assertEquals(2, pageMemory.invalidate(GRP_ID, PARTITION_ID));
+
+ Set<DirtyFullPageId> dirtyPagesAfterInvalidation = Set.of(createDirtyPage(pageMemory), createDirtyPage(pageMemory));
+ assertThat(pageMemory.dirtyPages(), equalTo(union(dirtyPages, dirtyPagesAfterInvalidation)));
} finally {
checkpointManager.checkpointTimeoutLock().checkpointReadUnlock();
}
@@ -460,18 +466,18 @@ public class PersistentPageMemoryNoLoadTest extends AbstractPageMemoryNoLoadSelf
);
}
- protected FullPageId createDirtyPage(PersistentPageMemory pageMemory) throws Exception {
+ private DirtyFullPageId createDirtyPage(PersistentPageMemory pageMemory) throws Exception {
FullPageId fullPageId = allocatePage(pageMemory);
long page = pageMemory.acquirePage(fullPageId.groupId(), fullPageId.pageId());
try {
writePage(pageMemory, fullPageId, page, 100);
+
+ return new DirtyFullPageId(fullPageId.pageId(), fullPageId.groupId(), partitionGeneration(page));
} finally {
pageMemory.releasePage(fullPageId.groupId(), fullPageId.pageId(), page);
}
-
- return fullPageId;
}
private static long[] defaultSegmentSizes() {
@@ -596,7 +602,7 @@ public class PersistentPageMemoryNoLoadTest extends AbstractPageMemoryNoLoadSelf
// Absolute memory pointer to page with header.
long absPtr = mem.acquirePage(fullPageId.groupId(), fullPageId.pageId());
- assertEquals(1, PageHeader.partitionGeneration(absPtr));
+ assertEquals(1, partitionGeneration(absPtr));
});
}
@@ -610,7 +616,7 @@ public class PersistentPageMemoryNoLoadTest extends AbstractPageMemoryNoLoadSelf
// Absolute memory pointer to page with header.
long absPtr = mem.acquirePage(fullPageId.groupId(), fullPageId.pageId());
- assertEquals(2, PageHeader.partitionGeneration(absPtr));
+ assertEquals(2, partitionGeneration(absPtr));
});
}
@@ -634,7 +640,7 @@ public class PersistentPageMemoryNoLoadTest extends AbstractPageMemoryNoLoadSelf
// Absolute memory pointer to page with header.
long absPtr = mem.acquirePage(fullPageId.groupId(), fullPageId.pageId());
- assertEquals(2, PageHeader.partitionGeneration(absPtr));
+ assertEquals(2, partitionGeneration(absPtr));
});
}