You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2017/04/24 17:13:11 UTC
ignite git commit: IGNITE-5024 - Fixing tests
Repository: ignite
Updated Branches:
refs/heads/ignite-5024 777bd7763 -> 6746d8261
IGNITE-5024 - Fixing tests
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6746d826
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6746d826
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6746d826
Branch: refs/heads/ignite-5024
Commit: 6746d82618343dcd1e57dadec337089476f451b0
Parents: 777bd77
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Mon Apr 24 20:14:01 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Mon Apr 24 20:14:01 2017 +0300
----------------------------------------------------------------------
.../pagemem/impl/PageMemoryNoStoreImpl.java | 122 ++++++++++++++++++-
.../IgniteCacheDatabaseSharedManager.java | 7 +-
.../evict/FairFifoPageEvictionTracker.java | 6 +-
.../evict/PageAbstractEvictionTracker.java | 89 ++------------
.../evict/Random2LruPageEvictionTracker.java | 4 +-
.../evict/RandomLruPageEvictionTracker.java | 4 +-
.../paged/PageEvictionAbstractTest.java | 2 +
7 files changed, 140 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6746d826/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
index d983790..63abdb8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.pagemem.impl;
import java.io.Closeable;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Arrays;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -143,6 +144,9 @@ public class PageMemoryNoStoreImpl implements PageMemory {
private OffheapReadWriteLock rwLock;
/** */
+ private final int totalPages;
+
+ /** */
private final boolean trackAcquiredPages;
/**
@@ -176,6 +180,8 @@ public class PageMemoryNoStoreImpl implements PageMemory {
assert sysPageSize % 8 == 0 : sysPageSize;
+ totalPages = (int)(memPlcCfg.getMaxSize() / sysPageSize);
+
// TODO configure concurrency level.
rwLock = new OffheapReadWriteLock(128);
}
@@ -337,6 +343,13 @@ public class PageMemoryNoStoreImpl implements PageMemory {
}
/**
+ * @return Total number of pages may be allocated for this instance.
+ */
+ public int totalPages() {
+ return totalPages;
+ }
+
+ /**
* @return Total number of acquired pages.
*/
public long acquiredPages() {
@@ -460,19 +473,64 @@ public class PageMemoryNoStoreImpl implements PageMemory {
}
/** {@inheritDoc} */
- @Override public void writeUnlock(int cacheId, long pageId, long page,
+ @Override public void writeUnlock(
+ int cacheId,
+ long pageId,
+ long page,
Boolean walPlc,
- boolean dirtyFlag) {
+ boolean dirtyFlag
+ ) {
long actualId = PageIO.getPageId(page + PAGE_OVERHEAD);
+
rwLock.writeUnlock(page + LOCK_OFFSET, PageIdUtils.tag(actualId));
}
+ /** {@inheritDoc} */
@Override public boolean isDirty(int cacheId, long pageId, long page) {
// always false for page no store.
return false;
}
/**
+ * @param pageIdx Page index.
+ * @return Total page sequence number.
+ */
+ public int pageSequenceNumber(int pageIdx) {
+ Segment seg = segment(pageIdx);
+
+ return seg.sequenceNumber(pageIdx);
+ }
+
+ /**
+ * @param seqNo Page sequence number.
+ * @return Page index.
+ */
+ public int pageIndex(int seqNo) {
+ Segment[] segs = segments;
+
+ int low = 0, high = segs.length - 1;
+
+ while (low <= high) {
+ int mid = (low + high) >>> 1;
+
+ Segment seg = segs[mid];
+
+ int cmp = seg.containsPageBySequence(seqNo);
+
+ if (cmp < 0)
+ high = mid - 1;
+ else if (cmp > 0) {
+ low = mid + 1;
+ }
+ else
+ return seg.pageIndex(seqNo);
+ }
+
+ throw new IgniteException("Allocated page must always be present in one of the segments [seqNo=" + seqNo +
+ ", segments=" + Arrays.toString(segs) + ']');
+ }
+
+ /**
* @param pageId Page ID to release.
*/
private void releaseFreePage(long pageId) {
@@ -562,7 +620,9 @@ public class PageMemoryNoStoreImpl implements PageMemory {
if (oldRef != null)
System.arraycopy(oldRef, 0, newRef, 0, oldRef.length);
- Segment allocated = new Segment(newRef.length - 1, region);
+ Segment lastSeg = oldRef == null ? null : oldRef[oldRef.length - 1];
+
+ Segment allocated = new Segment(newRef.length - 1, region, lastSeg == null ? 0 : lastSeg.sumPages());
allocated.init();
@@ -595,15 +655,23 @@ public class PageMemoryNoStoreImpl implements PageMemory {
private long pagesBase;
/** */
+ private int pagesInPrevSegments;
+
+ /** */
+ private int maxPages;
+
+ /** */
private final AtomicInteger acquiredPages;
/**
* @param idx Index.
* @param region Memory region to use.
+ * @param pagesInPrevSegments Number of pages in previously allocated segments.
*/
- private Segment(int idx, DirectMemoryRegion region) {
+ private Segment(int idx, DirectMemoryRegion region, int pagesInPrevSegments) {
this.idx = idx;
this.region = region;
+ this.pagesInPrevSegments = pagesInPrevSegments;
acquiredPages = new AtomicInteger();
}
@@ -622,6 +690,10 @@ public class PageMemoryNoStoreImpl implements PageMemory {
pagesBase = (base + 7) & ~0x7;
GridUnsafe.putLong(lastAllocatedIdxPtr, 0);
+
+ long limit = region.address() + region.size();
+
+ maxPages = (int)((limit - pagesBase) / sysPageSize);
}
/**
@@ -658,6 +730,23 @@ public class PageMemoryNoStoreImpl implements PageMemory {
}
/**
+ * @param pageIdx Page index with encoded segment.
+ * @return Absolute page sequence number.
+ */
+ private int sequenceNumber(int pageIdx) {
+ pageIdx &= IDX_MASK;
+
+ return pagesInPrevSegments + pageIdx;
+ }
+
+ /**
+ * @return Page sequence number upper bound.
+ */
+ private int sumPages() {
+ return pagesInPrevSegments + maxPages;
+ }
+
+ /**
* @return Total number of loaded pages for the segment.
*/
private int allocatedPages() {
@@ -680,7 +769,7 @@ public class PageMemoryNoStoreImpl implements PageMemory {
long limit = region.address() + region.size();
while (true) {
- long lastIdx = GridUnsafe.getLong(lastAllocatedIdxPtr);
+ long lastIdx = GridUnsafe.getLongVolatile(null, lastAllocatedIdxPtr);
// Check if we have enough space to allocate a page.
if (pagesBase + (lastIdx + 1) * sysPageSize > limit)
@@ -707,5 +796,28 @@ public class PageMemoryNoStoreImpl implements PageMemory {
}
}
}
+
+ /**
+ * @param seqNo Page sequence number.
+ * @return {@code 0} if this segment contains the page with the given sequence number,
+ * {@code -1} if one of the previous segments contains the page with the given sequence number,
+ * {@code 1} if one of the next segments contains the page with the given sequence number.
+ */
+ public int containsPageBySequence(int seqNo) {
+ if (seqNo < pagesInPrevSegments)
+ return -1;
+ else if (seqNo < pagesInPrevSegments + maxPages)
+ return 0;
+ else
+ return 1;
+ }
+
+ /**
+ * @param seqNo Page sequence number.
+ * @return Page index
+ */
+ public int pageIndex(int seqNo) {
+ return PageIdUtils.pageIndex(fromSegmentIndex(idx, seqNo - pagesInPrevSegments));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6746d826/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java
index 877180a..ae0921e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java
@@ -347,7 +347,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
"[name=" + plcCfg.getName() + ", size=" + U.readableSize(plcCfg.getInitialSize(), true) + "]"
);
- if (plcCfg.getMaxSize() > plcCfg.getInitialSize())
+ if (plcCfg.getMaxSize() < plcCfg.getInitialSize())
throw new IgniteCheckedException("MemoryPolicy maxSize must not be smaller than " +
"initialSize: " + plcCfg.getName());
}
@@ -636,14 +636,15 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
PageMemory pageMem = createPageMemory(memProvider, memCfg, plcCfg, memMetrics);
- return new MemoryPolicy(pageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg, pageMem));
+ return new MemoryPolicy(pageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg,
+ (PageMemoryNoStoreImpl)pageMem));
}
/**
* @param plc Memory Policy Configuration.
* @param pageMem Page memory.
*/
- private PageEvictionTracker createPageEvictionTracker(MemoryPolicyConfiguration plc, PageMemory pageMem) {
+ private PageEvictionTracker createPageEvictionTracker(MemoryPolicyConfiguration plc, PageMemoryNoStoreImpl pageMem) {
if (Boolean.getBoolean("override.fair.fifo.page.eviction.tracker"))
return new FairFifoPageEvictionTracker(pageMem, plc, cctx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6746d826/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/FairFifoPageEvictionTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/FairFifoPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/FairFifoPageEvictionTracker.java
index 8847013..b7c6b57 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/FairFifoPageEvictionTracker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/FairFifoPageEvictionTracker.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.ignite.internal.processors.cache.database.evict;
import java.util.LinkedList;
@@ -21,7 +22,7 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.configuration.MemoryPolicyConfiguration;
import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
/**
@@ -36,7 +37,8 @@ public class FairFifoPageEvictionTracker extends PageAbstractEvictionTracker {
* @param plcCfg Memory policy configuration.
* @param sharedCtx Shared context.
*/
- public FairFifoPageEvictionTracker(PageMemory pageMem,
+ public FairFifoPageEvictionTracker(
+ PageMemoryNoStoreImpl pageMem,
MemoryPolicyConfiguration plcCfg,
GridCacheSharedContext sharedCtx) {
super(pageMem, plcCfg, sharedCtx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6746d826/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/PageAbstractEvictionTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/PageAbstractEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/PageAbstractEvictionTracker.java
index 510ffae..4a65aee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/PageAbstractEvictionTracker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/PageAbstractEvictionTracker.java
@@ -18,10 +18,9 @@ package org.apache.ignite.internal.processors.cache.database.evict;
import java.util.List;
import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.configuration.MemoryPolicyConfiguration;
import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
@@ -39,13 +38,13 @@ public abstract class PageAbstractEvictionTracker implements PageEvictionTracker
private static final int COMPACT_TS_SHIFT = 8; // Enough if grid works for less than 17 years.
/** Millis in day. */
- private final static int DAY = 24 * 60 * 60 * 1000;
+ private static final int DAY = 24 * 60 * 60 * 1000;
/** Page memory. */
- protected final PageMemory pageMem;
+ protected final PageMemoryNoStoreImpl pageMem;
/** Tracking array size. */
- final int trackingSize;
+ protected final int trackingSize;
/** Base compact timestamp. */
private final long baseCompactTs;
@@ -53,21 +52,13 @@ public abstract class PageAbstractEvictionTracker implements PageEvictionTracker
/** Shared context. */
private final GridCacheSharedContext sharedCtx;
- /* TODO: IGNITE-4921: Will be removed after segments refactoring >>>> */
- protected final int segBits;
- protected final int idxBits;
- protected final int segMask;
- protected final int idxMask;
- protected final int segmentPageCount;
- /* <<<< */
-
/**
* @param pageMem Page memory.
* @param plcCfg Memory policy configuration.
* @param sharedCtx Shared context.
*/
PageAbstractEvictionTracker(
- PageMemory pageMem,
+ PageMemoryNoStoreImpl pageMem,
MemoryPolicyConfiguration plcCfg,
GridCacheSharedContext sharedCtx
) {
@@ -75,33 +66,7 @@ public abstract class PageAbstractEvictionTracker implements PageEvictionTracker
this.sharedCtx = sharedCtx;
- MemoryConfiguration memCfg = sharedCtx.kernalContext().config().getMemoryConfiguration();
-
- /* TODO: IGNITE-4921: Will be removed after segments refactoring >>>> */
- int concurrencyLevel = memCfg.getConcurrencyLevel();
-
- if (concurrencyLevel < 1)
- concurrencyLevel = Runtime.getRuntime().availableProcessors();
-
- int pageSize = memCfg.getPageSize();
-
- long segSize = plcCfg.getMaxSize() / concurrencyLevel;
-
- if (segSize < 1024 * 1024)
- segSize = 1024 * 1024;
-
- segmentPageCount = (int)(segSize / pageSize);
-
- segBits = Integer.SIZE - Integer.numberOfLeadingZeros(concurrencyLevel - 1);
-
- idxBits = PageIdUtils.PAGE_IDX_SIZE - segBits;
-
- segMask = ~(-1 << segBits);
-
- idxMask = ~(-1 << idxBits);
- /* <<<< */
-
- trackingSize = segmentPageCount << segBits;
+ trackingSize = pageMem.totalPages();
baseCompactTs = (U.currentTimeMillis() - DAY) >> COMPACT_TS_SHIFT;
// We subtract day to avoid fail in case of daylight shift or timezone change.
@@ -190,15 +155,7 @@ public abstract class PageAbstractEvictionTracker implements PageEvictionTracker
* @return Position of page in tracking array.
*/
int trackingIdx(int pageIdx) {
- int inSegmentPageIdx = inSegmentPageIdx(pageIdx);
-
- assert inSegmentPageIdx < segmentPageCount : inSegmentPageIdx;
-
- int trackingIdx = segmentIdx(pageIdx) * segmentPageCount + inSegmentPageIdx;
-
- assert trackingIdx < trackingSize : trackingIdx;
-
- return trackingIdx;
+ return pageMem.pageSequenceNumber(pageIdx);
}
/**
@@ -208,36 +165,6 @@ public abstract class PageAbstractEvictionTracker implements PageEvictionTracker
* @return Page index.
*/
int pageIdx(int trackingIdx) {
- assert trackingIdx < trackingSize;
-
- long res = 0;
-
- long segIdx = trackingIdx / segmentPageCount;
- long pageIdx = trackingIdx % segmentPageCount;
-
- res = (res << segBits) | (segIdx & segMask);
- res = (res << idxBits) | (pageIdx & idxMask);
-
- assert (res & (-1L << 32)) == 0 : res;
-
- return (int)res;
- }
-
- /* TODO: IGNITE-4921: Will be removed after segments refactoring >>>> */
- /**
- * @param pageIdx Page index.
- * @return Number of segment.
- */
- private int segmentIdx(int pageIdx) {
- return (pageIdx >> idxBits) & segMask;
- }
-
- /**
- * @param pageIdx Page index.
- * @return Number of page inside segment.
- */
- private int inSegmentPageIdx(int pageIdx) {
- return pageIdx & idxMask;
+ return pageMem.pageIndex(trackingIdx);
}
- /* <<<< */
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6746d826/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/Random2LruPageEvictionTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/Random2LruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/Random2LruPageEvictionTracker.java
index 870b4b1..21ebba1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/Random2LruPageEvictionTracker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/Random2LruPageEvictionTracker.java
@@ -23,7 +23,7 @@ import org.apache.ignite.IgniteLogger;
import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.configuration.MemoryPolicyConfiguration;
import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.util.GridUnsafe;
import org.apache.ignite.internal.util.typedef.internal.LT;
@@ -54,7 +54,7 @@ public class Random2LruPageEvictionTracker extends PageAbstractEvictionTracker {
* @param sharedCtx Shared context.
*/
public Random2LruPageEvictionTracker(
- PageMemory pageMem,
+ PageMemoryNoStoreImpl pageMem,
MemoryPolicyConfiguration plcCfg,
GridCacheSharedContext<?, ?> sharedCtx
) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/6746d826/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/RandomLruPageEvictionTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/RandomLruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/RandomLruPageEvictionTracker.java
index 169ec7e..d241148 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/RandomLruPageEvictionTracker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/RandomLruPageEvictionTracker.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.ignite.internal.processors.cache.database.evict;
import java.util.concurrent.ThreadLocalRandom;
@@ -24,6 +25,7 @@ import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.configuration.MemoryPolicyConfiguration;
import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.util.GridUnsafe;
import org.apache.ignite.internal.util.typedef.internal.LT;
@@ -58,7 +60,7 @@ public class RandomLruPageEvictionTracker extends PageAbstractEvictionTracker {
MemoryPolicyConfiguration plcCfg,
GridCacheSharedContext<?, ?> sharedCtx
) {
- super(pageMem, plcCfg, sharedCtx);
+ super((PageMemoryNoStoreImpl)pageMem, plcCfg, sharedCtx);
MemoryConfiguration memCfg = sharedCtx.kernalContext().config().getMemoryConfiguration();
http://git-wip-us.apache.org/repos/asf/ignite/blob/6746d826/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java
index 6fb4397..fb64ca5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionAbstractTest.java
@@ -79,6 +79,8 @@ public class PageEvictionAbstractTest extends GridCommonAbstractTest {
MemoryPolicyConfiguration plc = new MemoryPolicyConfiguration();
+ // This will test additional segment allocation.
+ plc.setInitialSize(SIZE / 2);
plc.setMaxSize(SIZE);
plc.setEmptyPagesPoolSize(EMPTY_PAGES_POOL_SIZE);
plc.setEvictionThreshold(EVICTION_THRESHOLD);