You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2019/12/30 15:17:13 UTC
[ignite] branch master updated: IGNITE-12491 Dirty pages count
calculation optimized.
This is an automated email from the ASF dual-hosted git repository.
agura pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new c97b08e IGNITE-12491 Dirty pages count calculation optimized.
c97b08e is described below
commit c97b08e672e4bfc3ae01e7bed190cd08c2f78ca1
Author: ibessonov <be...@gmail.com>
AuthorDate: Mon Dec 30 18:11:15 2019 +0300
IGNITE-12491 Dirty pages count calculation optimized.
Signed-off-by: Andrey Gura <ag...@apache.org>
---
.../GridCacheDatabaseSharedManager.java | 19 ------
.../cache/persistence/pagemem/PageMemoryEx.java | 2 +-
.../cache/persistence/pagemem/PageMemoryImpl.java | 67 +++++++++++++---------
3 files changed, 42 insertions(+), 46 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 270f519..061f4b6 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -1729,25 +1729,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
checkpointLock.readLock().unlock();
- if (checkpointer != null) {
- Collection<DataRegion> dataRegs = context().database().dataRegions();
-
- if (dataRegs != null) {
- for (DataRegion dataReg : dataRegs) {
- if (!dataReg.config().isPersistenceEnabled())
- continue;
-
- PageMemoryEx mem = (PageMemoryEx)dataReg.pageMemory();
-
- if (mem != null && !mem.safeToUpdate()) {
- checkpointer.wakeupForCheckpoint(0, "too many dirty pages");
-
- break;
- }
- }
- }
- }
-
if (ASSERTION_ENABLED)
CHECKPOINT_LOCK_HOLD_COUNT.set(CHECKPOINT_LOCK_HOLD_COUNT.get() - 1);
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
index 8465783..3baf1da 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
@@ -99,7 +99,7 @@ public interface PageMemoryEx extends PageMemory {
/**
* Heuristic method which allows a thread to check if it safe to start memory struture modifications
- * in regard with checkpointing.
+ * in regard with checkpointing. May return false-negative result during or after partition eviction.
*
* @return {@code False} if there are too many dirty pages and a thread should wait for a
* checkpoint to begin.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 2f90d41..7d29543 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -32,6 +32,8 @@ import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
@@ -90,6 +92,7 @@ import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.spi.encryption.noop.NoopEncryptionSpi;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
+import org.jetbrains.annotations.TestOnly;
import static java.lang.Boolean.FALSE;
import static java.lang.Boolean.TRUE;
@@ -206,6 +209,9 @@ public class PageMemoryImpl implements PageMemoryEx {
/** Segments array. */
private volatile Segment[] segments;
+ /** @see #safeToUpdate() */
+ private final AtomicBoolean safeToUpdate = new AtomicBoolean(true);
+
/** Lock for segments changes. */
private final Object segmentsLock = new Object();
@@ -906,8 +912,10 @@ public class PageMemoryImpl implements PageMemoryEx {
Collection<FullPageId> dirtyPages = seg.dirtyPages;
- if (dirtyPages != null)
- dirtyPages.remove(new FullPageId(pageId, grpId));
+ if (dirtyPages != null) {
+ if (dirtyPages.remove(new FullPageId(pageId, grpId)))
+ seg.dirtyPagesCntr.decrementAndGet();
+ }
return relPtr;
}
@@ -1034,11 +1042,8 @@ public class PageMemoryImpl implements PageMemoryEx {
/** {@inheritDoc} */
@Override public boolean safeToUpdate() {
- if (segments != null) {
- for (Segment segment : segments)
- if (!segment.safeToUpdate())
- return false;
- }
+ if (segments != null)
+ return safeToUpdate.get();
return true;
}
@@ -1109,8 +1114,11 @@ public class PageMemoryImpl implements PageMemoryEx {
seg.checkpointPages = new CheckpointPages(dirtyPages, allowToReplace);
seg.dirtyPages = new GridConcurrentHashSet<>();
+ seg.dirtyPagesCntr.set(0);
}
+ safeToUpdate.set(true);
+
memMetrics.resetDirtyPages();
if (throttlingPlc != ThrottlingPolicy.DISABLED)
@@ -1659,7 +1667,7 @@ public class PageMemoryImpl implements PageMemoryEx {
assert PageIO.getCrc(page + PAGE_OVERHEAD) == 0; //TODO GG-11480
if (markDirty)
- setDirty(fullId, page, markDirty, false);
+ setDirty(fullId, page, true, false);
beforeReleaseWrite(fullId, page + PAGE_OVERHEAD, pageWalRec);
}
@@ -1799,17 +1807,26 @@ public class PageMemoryImpl implements PageMemoryEx {
assert stateChecker.checkpointLockIsHeldByThread();
if (!wasDirty || forceAdd) {
- boolean added = segment(pageId.groupId(), pageId.pageId()).dirtyPages.add(pageId);
+ Segment seg = segment(pageId.groupId(), pageId.pageId());
+
+ if (seg.dirtyPages.add(pageId)) {
+ long dirtyPagesCnt = seg.dirtyPagesCntr.incrementAndGet();
+
+ if (dirtyPagesCnt >= seg.maxDirtyPages)
+ safeToUpdate.set(false);
- if (added)
memMetrics.incrementDirtyPages();
+ }
}
}
else {
- boolean rmv = segment(pageId.groupId(), pageId.pageId()).dirtyPages.remove(pageId);
+ Segment seg = segment(pageId.groupId(), pageId.pageId());
+
+ if (seg.dirtyPages.remove(pageId)) {
+ seg.dirtyPagesCntr.decrementAndGet();
- if (rmv)
memMetrics.decrementDirtyPages();
+ }
}
}
@@ -1892,9 +1909,10 @@ public class PageMemoryImpl implements PageMemoryEx {
*
* @return Collection of all page IDs marked as dirty.
*/
+ @TestOnly
public Collection<FullPageId> dirtyPages() {
if (segments == null)
- return Collections.EMPTY_SET;
+ return Collections.emptySet();
Collection<FullPageId> res = new HashSet<>((int)loadedPages());
@@ -1935,11 +1953,14 @@ public class PageMemoryImpl implements PageMemoryEx {
/** Pages marked as dirty since the last checkpoint. */
private volatile Collection<FullPageId> dirtyPages = new GridConcurrentHashSet<>();
+ /** Atomic size counter for {@link #dirtyPages}. Used for {@link PageMemoryImpl#safeToUpdate()} calculation. */
+ private final AtomicLong dirtyPagesCntr = new AtomicLong();
+
/** Wrapper of pages of current checkpoint. */
private volatile CheckpointPages checkpointPages;
/** */
- private final int maxDirtyPages;
+ private final long maxDirtyPages;
/** Initial partition generation. */
private static final int INIT_PART_GENERATION = 1;
@@ -1980,8 +2001,8 @@ public class PageMemoryImpl implements PageMemoryEx {
pool = new PagePool(idx, poolRegion, sysPageSize, rwLock);
maxDirtyPages = throttlingPlc != ThrottlingPolicy.DISABLED
- ? pool.pages() * 3 / 4
- : Math.min(pool.pages() * 2 / 3, cpPoolPages);
+ ? pool.pages() * 3L / 4
+ : Math.min(pool.pages() * 2L / 3, cpPoolPages);
}
/**
@@ -1999,13 +2020,6 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/**
- *
- */
- private boolean safeToUpdate() {
- return dirtyPages.size() < maxDirtyPages;
- }
-
- /**
* @param dirtyRatioThreshold Throttle threshold.
*/
private boolean shouldThrottle(double dirtyRatioThreshold) {
@@ -2016,7 +2030,7 @@ public class PageMemoryImpl implements PageMemoryEx {
* @return dirtyRatio to be compared with Throttle threshold.
*/
private double getDirtyPagesRatio() {
- return ((double)dirtyPages.size()) / pages();
+ return dirtyPagesCntr.doubleValue() / pages();
}
/**
@@ -2393,7 +2407,7 @@ public class PageMemoryImpl implements PageMemoryEx {
throw new IgniteOutOfMemoryException("Failed to find a page for eviction [segmentCapacity=" + cap +
", loaded=" + loadedPages.size() +
", maxDirtyPages=" + maxDirtyPages +
- ", dirtyPages=" + dirtyPages.size() +
+ ", dirtyPages=" + dirtyPagesCntr +
", cpPages=" + (checkpointPages == null ? 0 : checkpointPages.size()) +
", pinnedInSegment=" + pinnedCnt +
", failedToPrepare=" + failToPrepare +
@@ -2586,7 +2600,8 @@ public class PageMemoryImpl implements PageMemoryEx {
if (rmvDirty) {
FullPageId fullId = PageHeader.fullPageId(absPtr);
- seg.dirtyPages.remove(fullId);
+ if (seg.dirtyPages.remove(fullId))
+ seg.dirtyPagesCntr.decrementAndGet();
}
GridUnsafe.setMemory(absPtr + PAGE_OVERHEAD, pageSize, (byte)0);