You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2017/06/11 20:03:47 UTC
[21/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
new file mode 100644
index 0000000..205e0fc
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
@@ -0,0 +1,951 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReferenceArray;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.store.PageStore;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager;
+import org.apache.ignite.internal.processors.cache.IgniteRebalanceIterator;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseListImpl;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.junit.Assert;
+
+/**
+ *
+ */
+public class WalRecoveryTxLogicalRecordsTest extends GridCommonAbstractTest {
+ /** Cache name. */
+ private static final String CACHE_NAME = "cache";
+
+ /** Cache 2 name. */
+ private static final String CACHE2_NAME = "cache2";
+
+ /** */
+ public static final int PARTS = 32;
+
+ /** */
+ public static final int WAL_HIST_SIZE = 30;
+
+ /** */
+ private int pageSize = 4 * 1024;
+
+ /** */
+ private CacheConfiguration<?, ?> extraCcfg;
+
+ /** */
+ private Long checkpointFreq;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ CacheConfiguration<Integer, IndexedValue> ccfg = new CacheConfiguration<>(CACHE_NAME);
+
+ ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, PARTS));
+ ccfg.setIndexedTypes(Integer.class, IndexedValue.class);
+
+ if (extraCcfg != null)
+ cfg.setCacheConfiguration(ccfg, new CacheConfiguration<>(extraCcfg));
+ else
+ cfg.setCacheConfiguration(ccfg);
+
+ MemoryConfiguration dbCfg = new MemoryConfiguration();
+
+ dbCfg.setPageSize(pageSize);
+
+ cfg.setMemoryConfiguration(dbCfg);
+
+ PersistentStoreConfiguration pCfg = new PersistentStoreConfiguration();
+
+ pCfg.setWalHistorySize(WAL_HIST_SIZE);
+
+ if (checkpointFreq != null)
+ pCfg.setCheckpointingFrequency(checkpointFreq);
+
+ cfg.setPersistentStoreConfiguration(pCfg);
+
+ cfg.setMarshaller(null);
+
+ BinaryConfiguration binCfg = new BinaryConfiguration();
+
+ binCfg.setCompactFooter(false);
+
+ cfg.setBinaryConfiguration(binCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testWalTxSimple() throws Exception {
+ Ignite ignite = startGrid();
+
+ try {
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ dbMgr.enableCheckpoints(false).get();
+
+ IgniteCache<Integer, IndexedValue> cache = ignite.cache(CACHE_NAME);
+
+ int txCnt = 100;
+
+ int keysPerTx = 10;
+
+ for (int i = 0; i < txCnt; i++) {
+ try (Transaction tx = ignite.transactions().txStart()) {
+ for (int j = 0; j < keysPerTx; j++) {
+ int k = i * keysPerTx + j;
+
+ cache.put(k, new IndexedValue(k));
+ }
+
+ tx.commit();
+ }
+ }
+
+ for (int i = 0; i < txCnt; i++) {
+ for (int j = 0; j < keysPerTx; j++) {
+ int k = i * keysPerTx + j;
+
+ assertEquals(k, cache.get(k).value());
+ }
+ }
+
+ stopGrid();
+
+ ignite = startGrid();
+
+ cache = ignite.cache(CACHE_NAME);
+
+ for (int i = 0; i < txCnt; i++) {
+ for (int j = 0; j < keysPerTx; j++) {
+ int k = i * keysPerTx + j;
+
+ assertEquals(k, cache.get(k).value());
+ }
+ }
+
+ for (int i = 0; i < txCnt; i++) {
+ for (int j = 0; j < keysPerTx; j++) {
+ int k = i * keysPerTx + j;
+
+ QueryCursor<List<?>> cur = cache.query(
+ new SqlFieldsQuery("select sVal from IndexedValue where iVal=?").setArgs(k));
+
+ List<List<?>> vals = cur.getAll();
+
+ assertEquals(vals.size(), 1);
+ assertEquals("string-" + k, vals.get(0).get(0));
+ }
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testWalRecoveryRemoves() throws Exception {
+ Ignite ignite = startGrid();
+
+ try {
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ IgniteCache<Integer, IndexedValue> cache = ignite.cache(CACHE_NAME);
+
+ int txCnt = 100;
+
+ int keysPerTx = 10;
+
+ for (int i = 0; i < txCnt; i++) {
+ try (Transaction tx = ignite.transactions().txStart()) {
+ for (int j = 0; j < keysPerTx; j++) {
+ int k = i * keysPerTx + j;
+
+ cache.put(k, new IndexedValue(k));
+ }
+
+ tx.commit();
+ }
+ }
+
+ for (int i = 0; i < txCnt; i++) {
+ for (int j = 0; j < keysPerTx; j++) {
+ int k = i * keysPerTx + j;
+
+ assertEquals(k, cache.get(k).value());
+ }
+ }
+
+ dbMgr.waitForCheckpoint("test");
+ dbMgr.enableCheckpoints(false).get();
+
+ for (int i = 0; i < txCnt / 2; i++) {
+ try (Transaction tx = ignite.transactions().txStart()) {
+ for (int j = 0; j < keysPerTx; j++) {
+ int k = i * keysPerTx + j;
+
+ cache.remove(k);
+ }
+
+ tx.commit();
+ }
+ }
+
+ stopGrid();
+
+ ignite = startGrid();
+
+ cache = ignite.cache(CACHE_NAME);
+
+ for (int i = 0; i < txCnt; i++) {
+ for (int j = 0; j < keysPerTx; j++) {
+ int k = i * keysPerTx + j;
+
+ QueryCursor<List<?>> cur = cache.query(
+ new SqlFieldsQuery("select sVal from IndexedValue where iVal=?").setArgs(k));
+
+ List<List<?>> vals = cur.getAll();
+
+ if (i < txCnt / 2) {
+ assertNull(cache.get(k));
+ assertTrue(F.isEmpty(vals));
+ }
+ else {
+ assertEquals(k, cache.get(k).value());
+
+ assertEquals(1, vals.size());
+ assertEquals("string-" + k, vals.get(0).get(0));
+ }
+ }
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRebalanceIterator() throws Exception {
+ extraCcfg = new CacheConfiguration(CACHE2_NAME);
+ extraCcfg.setAffinity(new RendezvousAffinityFunction(false, PARTS));
+
+ Ignite ignite = startGrid();
+
+ try {
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ dbMgr.waitForCheckpoint("test");
+
+ // This number depends on wal history size.
+ int entries = 25;
+
+ IgniteCache<Integer, Integer> cache = ignite.cache(CACHE_NAME);
+ IgniteCache<Integer, Integer> cache2 = ignite.cache(CACHE2_NAME);
+
+ for (int i = 0; i < entries; i++) {
+ // Put to partition 0.
+ cache.put(i * PARTS, i * PARTS);
+
+ // Put to partition 1.
+ cache.put(i * PARTS + 1, i * PARTS + 1);
+
+ // Put to another cache.
+ cache2.put(i, i);
+
+ dbMgr.waitForCheckpoint("test");
+ }
+
+ for (int i = 0; i < entries; i++) {
+ assertEquals((Integer)(i * PARTS), cache.get(i * PARTS));
+ assertEquals((Integer)(i * PARTS + 1), cache.get(i * PARTS + 1));
+ assertEquals((Integer)(i), cache2.get(i));
+ }
+
+ GridCacheContext<Object, Object> cctx = ((IgniteEx)ignite).context().cache().cache(CACHE_NAME).context();
+ IgniteCacheOffheapManager offh = cctx.offheap();
+ AffinityTopologyVersion topVer = cctx.affinity().affinityTopologyVersion();
+
+ for (int i = 0; i < entries; i++) {
+ try (IgniteRebalanceIterator it = offh.rebalanceIterator(0, topVer, (long)i)) {
+ assertTrue("Not historical for iteration: " + i, it.historical());
+
+ assertNotNull(it);
+
+ for (int j = i; j < entries; j++) {
+ assertTrue("i=" + i + ", j=" + j, it.hasNextX());
+
+ CacheDataRow row = it.next();
+
+ assertEquals(j * PARTS, (int)row.key().value(cctx.cacheObjectContext(), false));
+ assertEquals(j * PARTS, (int)row.value().value(cctx.cacheObjectContext(), false));
+ }
+
+ assertFalse(it.hasNext());
+ }
+
+ try (IgniteRebalanceIterator it = offh.rebalanceIterator(1, topVer, (long)i)) {
+ assertNotNull(it);
+
+ assertTrue("Not historical for iteration: " + i, it.historical());
+
+ for (int j = i; j < entries; j++) {
+ assertTrue(it.hasNextX());
+
+ CacheDataRow row = it.next();
+
+ assertEquals(j * PARTS + 1, (int)row.key().value(cctx.cacheObjectContext(), false));
+ assertEquals(j * PARTS + 1, (int)row.value().value(cctx.cacheObjectContext(), false));
+ }
+
+ assertFalse(it.hasNext());
+ }
+ }
+
+ stopAllGrids();
+
+ // Check that iterator is valid after restart.
+ ignite = startGrid();
+
+ cctx = ((IgniteEx)ignite).context().cache().cache(CACHE_NAME).context();
+ offh = cctx.offheap();
+ topVer = cctx.affinity().affinityTopologyVersion();
+
+ for (int i = 0; i < entries; i++) {
+ long start = System.currentTimeMillis();
+
+ try (IgniteRebalanceIterator it = offh.rebalanceIterator(0, topVer, (long)i)) {
+ long end = System.currentTimeMillis();
+
+ info("Time to get iterator: " + (end - start));
+
+ assertTrue("Not historical for iteration: " + i, it.historical());
+
+ assertNotNull(it);
+
+ start = System.currentTimeMillis();
+
+ for (int j = i; j < entries; j++) {
+ assertTrue("i=" + i + ", j=" + j, it.hasNextX());
+
+ CacheDataRow row = it.next();
+
+ assertEquals(j * PARTS, (int)row.key().value(cctx.cacheObjectContext(), false));
+ assertEquals(j * PARTS, (int)row.value().value(cctx.cacheObjectContext(), false));
+ }
+
+ end = System.currentTimeMillis();
+
+ info("Time to iterate: " + (end - start));
+
+ assertFalse(it.hasNext());
+ }
+
+ try (IgniteRebalanceIterator it = offh.rebalanceIterator(1, topVer, (long)i)) {
+ assertNotNull(it);
+
+ assertTrue("Not historical for iteration: " + i, it.historical());
+
+ for (int j = i; j < entries; j++) {
+ assertTrue(it.hasNextX());
+
+ CacheDataRow row = it.next();
+
+ assertEquals(j * PARTS + 1, (int)row.key().value(cctx.cacheObjectContext(), false));
+ assertEquals(j * PARTS + 1, (int)row.value().value(cctx.cacheObjectContext(), false));
+ }
+
+ assertFalse(it.hasNext());
+ }
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testCheckpointHistory() throws Exception {
+ Ignite ignite = startGrid();
+
+ try {
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ dbMgr.waitForCheckpoint("test");
+
+ // This number depends on wal history size.
+ int entries = WAL_HIST_SIZE * 2;
+
+ IgniteCache<Integer, Integer> cache = ignite.cache(CACHE_NAME);
+
+ for (int i = 0; i < entries; i++) {
+ // Put to partition 0.
+ cache.put(i * PARTS, i * PARTS);
+
+ // Put to partition 1.
+ cache.put(i * PARTS + 1, i * PARTS + 1);
+
+ dbMgr.waitForCheckpoint("test");
+ }
+
+ GridCacheDatabaseSharedManager.CheckpointHistory hist = dbMgr.checkpointHistory();
+
+ assertTrue(hist.checkpoints().size() <= WAL_HIST_SIZE);
+
+ File cpDir = dbMgr.checkpointDirectory();
+
+ File[] cpFiles = cpDir.listFiles();
+
+ assertTrue(cpFiles.length <= WAL_HIST_SIZE * 2);
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testWalAfterPreloading() throws Exception {
+ Ignite ignite = startGrid();
+
+ try {
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ dbMgr.enableCheckpoints(false).get();
+
+ int entries = 100;
+
+ try (IgniteDataStreamer<Integer, Integer> streamer = ignite.dataStreamer(CACHE_NAME)) {
+ for (int i = 0; i < entries; i++)
+ streamer.addData(i, i);
+ }
+
+ IgniteCache<Integer, Integer> cache = ignite.cache(CACHE_NAME);
+
+ for (int i = 0; i < entries; i++)
+ assertEquals(new Integer(i), cache.get(i));
+
+ stopGrid();
+
+ ignite = startGrid();
+
+ cache = ignite.cache(CACHE_NAME);
+
+ for (int i = 0; i < entries; i++)
+ assertEquals(new Integer(i), cache.get(i));
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRecoveryRandomPutRemove() throws Exception {
+ try {
+ pageSize = 1024;
+
+ extraCcfg = new CacheConfiguration(CACHE2_NAME);
+ extraCcfg.setAffinity(new RendezvousAffinityFunction(false, PARTS));
+
+ Ignite ignite = startGrid(0);
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ dbMgr.enableCheckpoints(false).get();
+
+ IgniteCache<Integer, IndexedValue> cache1 = ignite.cache(CACHE_NAME);
+ IgniteCache<Object, Object> cache2 = ignite.cache(CACHE2_NAME);
+
+ final int KEYS1 = 100;
+
+ for (int i = 0; i < KEYS1; i++)
+ cache1.put(i, new IndexedValue(i));
+
+ for (int i = 0; i < KEYS1; i++) {
+ if (i % 2 == 0)
+ cache1.remove(i);
+ }
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < KEYS1; i++) {
+ cache2.put(i, new byte[rnd.nextInt(512)]);
+
+ if (rnd.nextBoolean())
+ cache2.put(i, new byte[rnd.nextInt(512)]);
+
+ if (rnd.nextBoolean())
+ cache2.remove(i);
+ }
+
+ ignite.close();
+
+ ignite = startGrid(0);
+
+ ignite.cache(CACHE_NAME).put(1, new IndexedValue(0));
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRecoveryNoPageLost1() throws Exception {
+ recoveryNoPageLost(false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRecoveryNoPageLost2() throws Exception {
+ recoveryNoPageLost(true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRecoveryNoPageLost3() throws Exception {
+ try {
+ pageSize = 1024;
+ checkpointFreq = 100L;
+ extraCcfg = new CacheConfiguration(CACHE2_NAME);
+ extraCcfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ List<Integer> pages = null;
+
+ for (int iter = 0; iter < 5; iter++) {
+ log.info("Start node: " + iter);
+
+ Ignite ignite = startGrid(0);
+
+ if (pages != null) {
+ List<Integer> curPags = allocatedPages(ignite, CACHE2_NAME);
+
+ assertEquals("Iter = " + iter, pages, curPags);
+ }
+
+ final IgniteCache<Integer, Object> cache = ignite.cache(CACHE2_NAME);
+
+ final int ops = ThreadLocalRandom.current().nextInt(10) + 10;
+
+ GridTestUtils.runMultiThreaded(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < ops; i++) {
+ Integer key = rnd.nextInt(1000);
+
+ cache.put(key, new byte[rnd.nextInt(512)]);
+
+ if (rnd.nextBoolean())
+ cache.remove(key);
+ }
+
+ return null;
+ }
+ }, 10, "update");
+
+ pages = allocatedPages(ignite, CACHE2_NAME);
+
+ Ignition.stop(ignite.name(), false); //will make checkpoint
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @param checkpoint Checkpoint enable flag.
+ * @throws Exception If failed.
+ */
+ private void recoveryNoPageLost(boolean checkpoint) throws Exception {
+ try {
+ pageSize = 1024;
+ extraCcfg = new CacheConfiguration(CACHE2_NAME);
+ extraCcfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ List<Integer> pages = null;
+
+ AtomicInteger cnt = new AtomicInteger();
+
+ for (int iter = 0; iter < 5; iter++) {
+ log.info("Start node: " + iter);
+
+ Ignite ignite = startGrid(0);
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ if (!checkpoint)
+ dbMgr.enableCheckpoints(false).get();
+
+ if (pages != null) {
+ List<Integer> curPags = allocatedPages(ignite, CACHE2_NAME);
+
+ assertEquals(pages, curPags);
+ }
+
+ IgniteCache<Integer, Object> cache = ignite.cache(CACHE2_NAME);
+
+ for (int i = 0; i < 128; i++)
+ cache.put(cnt.incrementAndGet(), new byte[256 + iter * 100]);
+
+ pages = allocatedPages(ignite, CACHE2_NAME);
+
+ ignite.close();
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @param ignite Node.
+ * @param cacheName Cache name.
+ * @return Allocated pages per-store.
+ * @throws Exception If failed.
+ */
+ private List<Integer> allocatedPages(Ignite ignite, String cacheName) throws Exception {
+ FilePageStoreManager storeMgr =
+ (FilePageStoreManager)((IgniteEx)ignite).context().cache().context().pageStore();
+
+ int parts = ignite.affinity(cacheName).partitions();
+
+ List<Integer> res = new ArrayList<>(parts);
+
+ for (int p = 0; p < parts; p++) {
+ PageStore store = storeMgr.getStore(CU.cacheId(cacheName), p);
+
+ store.sync();
+
+ res.add(store.pages());
+ }
+
+ PageStore store = storeMgr.getStore(CU.cacheId(cacheName), PageIdAllocator.INDEX_PARTITION);
+
+ store.sync();
+
+ res.add(store.pages());
+
+ return res;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testFreeListRecovery() throws Exception {
+ try {
+ pageSize = 1024;
+ extraCcfg = new CacheConfiguration(CACHE2_NAME);
+
+ Ignite ignite = startGrid(0);
+
+ IgniteCache<Integer, IndexedValue> cache1 = ignite.cache(CACHE_NAME);
+ IgniteCache<Object, Object> cache2 = ignite.cache(CACHE2_NAME);
+
+ final int KEYS1 = 2048;
+
+ for (int i = 0; i < KEYS1; i++)
+ cache1.put(i, new IndexedValue(i));
+
+ for (int i = 0; i < KEYS1; i++) {
+ if (i % 2 == 0)
+ cache1.remove(i);
+ }
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < KEYS1; i++) {
+ cache2.put(i, new byte[rnd.nextInt(512)]);
+
+ if (rnd.nextBoolean())
+ cache2.put(i, new byte[rnd.nextInt(512)]);
+
+ if (rnd.nextBoolean())
+ cache2.remove(i);
+ }
+
+ Map<Integer, T2<Map<Integer, long[]>, int[]>> cache1_1 = getFreeListData(ignite, CACHE_NAME);
+ Map<Integer, T2<Map<Integer, long[]>, int[]>> cache2_1 = getFreeListData(ignite, CACHE2_NAME);
+ T2<long[], Integer> rl1_1 = getReuseListData(ignite, CACHE_NAME);
+ T2<long[], Integer> rl2_1 = getReuseListData(ignite, CACHE2_NAME);
+
+ ignite.close();
+
+ ignite = startGrid(0);
+
+ cache1 = ignite.cache(CACHE_NAME);
+ cache2 = ignite.cache(CACHE2_NAME);
+
+ for (int i = 0; i < KEYS1; i++) {
+ cache1.get(i);
+ cache2.get(i);
+ }
+
+ Map<Integer, T2<Map<Integer, long[]>, int[]>> cache1_2 = getFreeListData(ignite, CACHE_NAME);
+ Map<Integer, T2<Map<Integer, long[]>, int[]>> cache2_2 = getFreeListData(ignite, CACHE2_NAME);
+ T2<long[], Integer> rl1_2 = getReuseListData(ignite, CACHE_NAME);
+ T2<long[], Integer> rl2_2 = getReuseListData(ignite, CACHE2_NAME);
+
+ checkEquals(cache1_1, cache1_2);
+ checkEquals(cache2_1, cache2_2);
+ checkEquals(rl1_1, rl1_2);
+ checkEquals(rl2_1, rl2_2);
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @param ignite Node.
+ * @param cacheName Cache name.
+ * @return Cache reuse list data.
+ */
+ private T2<long[], Integer> getReuseListData(Ignite ignite, String cacheName) {
+ GridCacheContext ctx = ((IgniteEx)ignite).context().cache().cache(cacheName).context();
+
+ ReuseListImpl reuseList = GridTestUtils.getFieldValue(ctx.offheap(), "reuseList");
+ PagesList.Stripe[] bucket = GridTestUtils.getFieldValue(reuseList, "bucket");
+
+ long[] ids = null;
+
+ if (bucket != null) {
+ ids = new long[bucket.length];
+
+ for (int i = 0; i < bucket.length; i++)
+ ids[i] = bucket[i].tailId;
+ }
+
+// AtomicIntegerArray cnts = GridTestUtils.getFieldValue(reuseList, PagesList.class, "cnts");
+// assertEquals(1, cnts.length());
+
+ return new T2<>(ids, 0);
+ }
+
+ /**
+ * @param rl1 Data 1 (before stop).
+ * @param rl2 Data 2 (after restore).
+ */
+ private void checkEquals(T2<long[], Integer> rl1, T2<long[], Integer> rl2) {
+ Assert.assertArrayEquals(rl1.get1(), rl2.get1());
+ assertEquals(rl1.get2(), rl2.get2());
+ }
+
+ /**
+ * @param partsLists1 Data 1 (before stop).
+ * @param partsLists2 Data 2 (after restore).
+ */
+ private void checkEquals(Map<Integer, T2<Map<Integer, long[]>, int[]>> partsLists1,
+ Map<Integer, T2<Map<Integer, long[]>, int[]>> partsLists2) {
+ assertEquals(partsLists1.size(), partsLists2.size());
+
+ for (Integer part : partsLists1.keySet()) {
+ T2<Map<Integer, long[]>, int[]> t1 = partsLists1.get(part);
+ T2<Map<Integer, long[]>, int[]> t2 = partsLists2.get(part);
+
+ Map<Integer, long[]> m1 = t1.get1();
+ Map<Integer, long[]> m2 = t2.get1();
+
+ assertEquals(m1.size(), m2.size());
+
+ for (Integer bucket : m1.keySet()) {
+ long tails1[] = m1.get(bucket);
+ long tails2[] = m2.get(bucket);
+
+ Assert.assertArrayEquals(tails1, tails2);
+ }
+
+ Assert.assertArrayEquals("Wrong counts [part=" + part + ']', t1.get2(), t2.get2());
+ }
+ }
+
+ /**
+ * @param ignite Node.
+ * @param cacheName Cache name.
+ * @return Cache free lists data.
+ */
+ private Map<Integer, T2<Map<Integer, long[]>, int[]>> getFreeListData(Ignite ignite, String cacheName) {
+ GridCacheContext ctx = ((IgniteEx)ignite).context().cache().cache(cacheName).context();
+
+ List<GridDhtLocalPartition> parts = ctx.topology().localPartitions();
+
+ assertTrue(!parts.isEmpty());
+ assertEquals(ctx.affinity().partitions(), parts.size());
+
+ Map<Integer, T2<Map<Integer, long[]>, int[]>> res = new HashMap<>();
+
+ boolean foundNonEmpty = false;
+ boolean foundTails = false;
+
+ for (GridDhtLocalPartition part : parts) {
+ FreeListImpl freeList = GridTestUtils.getFieldValue(part.dataStore(), "freeList");
+
+ if (freeList == null)
+ // Lazy store.
+ continue;
+
+ AtomicReferenceArray<PagesList.Stripe[]> buckets = GridTestUtils.getFieldValue(freeList,
+ FreeListImpl.class, "buckets");
+ //AtomicIntegerArray cnts = GridTestUtils.getFieldValue(freeList, PagesList.class, "cnts");
+
+ assertNotNull(buckets);
+ //assertNotNull(cnts);
+ assertTrue(buckets.length() > 0);
+ //assertEquals(cnts.length(), buckets.length());
+
+ Map<Integer, long[]> tailsPerBucket = new HashMap<>();
+
+ for (int i = 0; i < buckets.length(); i++) {
+ PagesList.Stripe[] tails = buckets.get(i);
+
+ long ids[] = null;
+
+ if (tails != null) {
+ ids = new long[tails.length];
+
+ for (int j = 0; j < tails.length; j++)
+ ids[j] = tails[j].tailId;
+ }
+
+ tailsPerBucket.put(i, ids);
+
+ if (tails != null) {
+ assertTrue(tails.length > 0);
+
+ foundTails = true;
+ }
+ }
+
+// int[] cntsPerBucket = new int[cnts.length()];
+//
+// for (int i = 0; i < cnts.length(); i++) {
+// cntsPerBucket[i] = cnts.get(i);
+//
+// if (cntsPerBucket[i] > 0)
+// foundNonEmpty = true;
+// }
+
+ res.put(part.id(), new T2<>(tailsPerBucket, (int[])null));
+ }
+
+ //assertTrue(foundNonEmpty);
+ assertTrue(foundTails);
+
+ return res;
+ }
+
+ /**
+ *
+ */
+ private static class IndexedValue {
+ /** */
+ @QuerySqlField(index = true)
+ private int iVal;
+
+ /** */
+ @QuerySqlField
+ private String sVal;
+
+ /**
+ * @param iVal Indexed value.
+ */
+ private IndexedValue(int iVal) {
+ this.iVal = iVal;
+ sVal = "string-" + iVal;
+ }
+
+ /**
+ * @return Value.
+ */
+ private int value() {
+ return iVal;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
new file mode 100644
index 0000000..303f14e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal.crc;
+
+import junit.framework.TestCase;
+import java.io.EOFException;
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
+
+/**
+ *
+ */
+public class IgniteDataIntegrityTests extends TestCase {
+ /** File input. */
+ private FileInput fileInput;
+
+ /** Random access file. */
+ private RandomAccessFile randomAccessFile;
+
+ /** {@inheritDoc} */
+ @Override protected void setUp() throws Exception {
+ super.setUp();
+
+ File file = File.createTempFile("integrity", "dat");
+ file.deleteOnExit();
+
+ randomAccessFile = new RandomAccessFile(file, "rw");
+
+ fileInput = new FileInput(randomAccessFile.getChannel(), ByteBuffer.allocate(1024));
+
+ PureJavaCrc32 pureJavaCrc32 = new PureJavaCrc32();
+
+ ByteBuffer buf = ByteBuffer.allocate(1024);
+ ThreadLocalRandom curr = ThreadLocalRandom.current();
+
+ for (int i = 0; i < 1024; i+=16) {
+ buf.putInt(curr.nextInt());
+ buf.putInt(curr.nextInt());
+ buf.putInt(curr.nextInt());
+ buf.position(i);
+ buf.putInt(PureJavaCrc32.calcCrc32(buf, 12));
+ }
+
+ randomAccessFile.write(buf.array());
+ randomAccessFile.getFD().sync();
+ }
+
+ /**
+ *
+ */
+ public void testSuccessfulPath() throws Exception {
+ checkIntegrity();
+ }
+
+ /**
+ *
+ */
+ public void testIntegrityViolationChecking() throws Exception {
+ toggleOneRandomBit(0, 1024 - 16);
+
+ try {
+ checkIntegrity();
+
+ fail();
+ } catch (IgniteDataIntegrityViolationException ex) {
+ //success
+ }
+ }
+
+ /**
+ *
+ */
+ public void testSkipingLastCorruptedEntry() throws Exception {
+ toggleOneRandomBit(1024 - 16, 1024);
+
+ try {
+ checkIntegrity();
+
+ fail();
+ } catch (EOFException ex) {
+ //success
+ }
+ }
+
+ /**
+ * @param rangeFrom Range from.
+ * @param rangeTo Range to.
+ */
+ private void toggleOneRandomBit(int rangeFrom, int rangeTo) throws IOException {
+ int pos = ThreadLocalRandom.current().nextInt(rangeFrom, rangeTo);
+ randomAccessFile.seek(pos);
+
+ byte b = randomAccessFile.readByte();
+
+ b ^= (1 << 3);
+
+ randomAccessFile.seek(pos);
+ randomAccessFile.writeByte(b);
+ randomAccessFile.getFD().sync();
+ }
+
+ /**
+ *
+ */
+ private void checkIntegrity() throws Exception {
+ randomAccessFile.seek(0);
+
+ for (int i = 0; i < 1024 / 16; i++) {
+ try(FileInput.Crc32CheckingFileInput in = fileInput.startRead(false)) {
+ in.readInt();
+ in.readInt();
+ in.readInt();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
new file mode 100644
index 0000000..6f58782
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
+import org.apache.ignite.internal.processors.database.BPlusTreeSelfTest;
+import org.apache.ignite.internal.util.typedef.CIX3;
+import org.apache.ignite.testframework.junits.GridTestKernalContext;
+
+/**
+ *
+ */
+public class BPlusTreePageMemoryImplTest extends BPlusTreeSelfTest {
+ /** {@inheritDoc} */
+ @Override protected PageMemory createPageMemory() throws Exception {
+ long[] sizes = new long[CPUS + 1];
+
+ for (int i = 0; i < sizes.length; i++)
+ sizes[i] = 1024 * MB / CPUS;
+
+ sizes[CPUS] = 10 * MB;
+
+ DirectMemoryProvider provider = new UnsafeMemoryProvider(log);
+
+ GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
+ new GridTestKernalContext(log),
+ null,
+ null,
+ null,
+ new NoOpPageStoreManager(),
+ new NoOpWALManager(),
+ new IgniteCacheDatabaseSharedManager(),
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null
+ );
+
+ PageMemory mem = new PageMemoryImpl(
+ provider, sizes,
+ sharedCtx,
+ PAGE_SIZE,
+ new CIX3<FullPageId, ByteBuffer, Integer>() {
+ @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuf, Integer tag) {
+ assert false : "No evictions should happen during the test";
+ }
+ },
+ new CIX3<Long, FullPageId, PageMemoryEx>(){
+ @Override public void applyx(Long aLong, FullPageId fullPageId, PageMemoryEx ex) {
+ }
+ },
+ new CheckpointLockStateChecker() {
+ @Override public boolean checkpointLockIsHeldByThread() {
+ return true;
+ }
+ },
+ new MemoryMetricsImpl(new MemoryPolicyConfiguration()));
+
+ mem.start();
+
+ return mem;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long acquiredPages() {
+ return ((PageMemoryImpl)pageMem).acquiredPages();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
new file mode 100644
index 0000000..b263d4f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
+import org.apache.ignite.internal.processors.database.BPlusTreeReuseSelfTest;
+import org.apache.ignite.internal.util.lang.GridInClosure3X;
+import org.apache.ignite.internal.util.typedef.CIX3;
+import org.apache.ignite.testframework.junits.GridTestKernalContext;
+
+/**
+ *
+ */
+public class BPlusTreeReuseListPageMemoryImplTest extends BPlusTreeReuseSelfTest {
+ /** {@inheritDoc} */
+ @Override protected PageMemory createPageMemory() throws Exception {
+ long[] sizes = new long[CPUS + 1];
+
+ for (int i = 0; i < sizes.length; i++)
+ sizes[i] = 1024 * MB / CPUS;
+
+ sizes[CPUS] = 10 * MB;
+
+ DirectMemoryProvider provider = new UnsafeMemoryProvider(log);
+
+ GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
+ new GridTestKernalContext(log),
+ null,
+ null,
+ null,
+ new NoOpPageStoreManager(),
+ new NoOpWALManager(),
+ new IgniteCacheDatabaseSharedManager(),
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null
+ );
+
+ PageMemory mem = new PageMemoryImpl(
+ provider, sizes,
+ sharedCtx,
+ PAGE_SIZE,
+ new CIX3<FullPageId, ByteBuffer, Integer>() {
+ @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuf, Integer tag) {
+ assert false : "No evictions should happen during the test";
+ }
+ },
+ new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
+ @Override public void applyx(Long page, FullPageId fullPageId, PageMemoryEx pageMem) {
+ }
+ }, new CheckpointLockStateChecker() {
+ @Override public boolean checkpointLockIsHeldByThread() {
+ return true;
+ }
+ },
+ new MemoryMetricsImpl(new MemoryPolicyConfiguration())
+ );
+
+ mem.start();
+
+ return mem;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long acquiredPages() {
+ return ((PageMemoryImpl)pageMem).acquiredPages();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java
new file mode 100644
index 0000000..1f29549
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+import org.apache.ignite.internal.mem.DirectMemoryRegion;
+import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.FullPageIdTable;
+import org.apache.ignite.internal.util.typedef.CI2;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.logger.java.JavaLogger;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class FullPageIdTableTest extends GridCommonAbstractTest {
+ /** */
+ private static final int CACHE_ID_RANGE = 10;
+
+ /** */
+ private static final int PAGE_ID_RANGE = 1000;
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRandomOperations() throws Exception {
+ long mem = FullPageIdTable.requiredMemory(CACHE_ID_RANGE * PAGE_ID_RANGE);
+
+ UnsafeMemoryProvider prov = new UnsafeMemoryProvider(new JavaLogger());
+
+ prov.initialize(new long[] {mem});
+
+ DirectMemoryRegion region = prov.nextRegion();
+
+ try {
+ long seed = U.currentTimeMillis();
+
+ info("Seed: " + seed + "L; //");
+
+ Random rnd = new Random(seed);
+
+ FullPageIdTable tbl = new FullPageIdTable(region.address(), region.size(), true);
+
+ Map<FullPageId, Long> check = new HashMap<>();
+
+ for (int i = 0; i < 10_000; i++) {
+ int cacheId = rnd.nextInt(CACHE_ID_RANGE) + 1;
+ int pageId = rnd.nextInt(PAGE_ID_RANGE);
+
+ FullPageId fullId = new FullPageId(pageId, cacheId);
+
+ boolean put = rnd.nextInt(3) != -1;
+
+ if (put) {
+ long val = rnd.nextLong();
+
+ tbl.put(cacheId, pageId, val, 0);
+ check.put(fullId, val);
+ }
+ else {
+ tbl.remove(cacheId, pageId, 0);
+ check.remove(fullId);
+ }
+
+ verifyLinear(tbl, check);
+
+ if (i > 0 && i % 1000 == 0)
+ info("Done: " + i);
+ }
+ }
+ finally {
+ prov.shutdown();
+ }
+ }
+
+ /**
+ * @param tbl Table to check.
+ * @param check Expected mapping.
+ */
+ private void verifyLinear(FullPageIdTable tbl, Map<FullPageId, Long> check) {
+ final Map<FullPageId, Long> collector = new HashMap<>();
+
+ tbl.visitAll(new CI2<FullPageId, Long>() {
+ @Override public void apply(FullPageId fullId, Long val) {
+ if (collector.put(fullId, val) != null)
+ throw new AssertionError("Duplicate full page ID mapping: " + fullId);
+ }
+ });
+
+ assertEquals("Size check failed", check.size(), collector.size());
+
+ for (Map.Entry<FullPageId, Long> entry : check.entrySet())
+ assertEquals("Mapping comparison failed for key: " + entry.getKey(),
+ entry.getValue(), collector.get(entry.getKey()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java
new file mode 100644
index 0000000..d9257bd
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
+import org.apache.ignite.internal.processors.database.MetadataStorageSelfTest;
+import org.apache.ignite.internal.util.lang.GridInClosure3X;
+import org.apache.ignite.internal.util.typedef.CIX3;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.GridTestKernalContext;
+
+/**
+ *
+ */
+public class MetadataStoragePageMemoryImplTest extends MetadataStorageSelfTest{
+ /** Make sure page is small enough to trigger multiple pages in a linked list. */
+ public static final int PAGE_SIZE = 1024;
+
+ /** */
+ private static File allocationPath;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ allocationPath = U.resolveWorkDirectory(U.defaultWorkDirectory(), "pagemem", false);
+ }
+
+ /**
+ * @param clean Clean flag. If {@code true}, will clean previous memory state and allocate
+ * new empty page memory.
+ * @return Page memory instance.
+ */
+ @Override protected PageMemory memory(boolean clean) throws Exception {
+ long[] sizes = new long[10];
+
+ for (int i = 0; i < sizes.length; i++)
+ sizes[i] = 1024 * 1024;
+
+ DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), allocationPath);
+
+ GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
+ new GridTestKernalContext(log),
+ null,
+ null,
+ null,
+ new NoOpPageStoreManager(),
+ new NoOpWALManager(),
+ new IgniteCacheDatabaseSharedManager(),
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null
+ );
+
+ return new PageMemoryImpl(
+ provider, sizes,
+ sharedCtx,
+ PAGE_SIZE,
+ new CIX3<FullPageId, ByteBuffer, Integer>() {
+ @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuf, Integer tag) {
+ assert false : "No evictions should happen during the test";
+ }
+ },
+ new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
+ @Override public void applyx(Long page, FullPageId fullId, PageMemoryEx pageMem) {
+ }
+ }, new CheckpointLockStateChecker() {
+ @Override public boolean checkpointLockIsHeldByThread() {
+ return true;
+ }
+ },
+ new MemoryMetricsImpl(new MemoryPolicyConfiguration()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
new file mode 100644
index 0000000..0ef593f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.lang.IgniteFuture;
+
+/**
+ *
+ */
+public class NoOpWALManager implements IgniteWriteAheadLogManager {
+ /** {@inheritDoc} */
+ @Override public boolean isAlwaysWriteFullPages() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isFullSync() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void resumeLogging(WALPointer ptr) throws IgniteCheckedException {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public WALPointer log(WALRecord entry) throws IgniteCheckedException, StorageException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void fsync(WALPointer ptr) throws IgniteCheckedException, StorageException {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public WALIterator replay(WALPointer start) throws IgniteCheckedException, StorageException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean reserve(WALPointer start) throws IgniteCheckedException {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void release(WALPointer start) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public int truncate(WALPointer ptr) {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean reserved(WALPointer ptr) {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void start(GridCacheSharedContext cctx) throws IgniteCheckedException {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop(boolean cancel) {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onKernalStart(boolean reconnect) throws IgniteCheckedException {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onKernalStop(boolean cancel) {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onDisconnected(IgniteFuture reconnectFut) {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void printMemoryStats() {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public int walArchiveSegments() {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageIdDistributionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageIdDistributionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageIdDistributionTest.java
new file mode 100644
index 0000000..1a8aaa4
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageIdDistributionTest.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import org.apache.ignite.internal.mem.DirectMemoryRegion;
+import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.FullPageIdTable;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.logger.java.JavaLogger;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class PageIdDistributionTest extends GridCommonAbstractTest {
+ /** */
+ private static final int[] CACHE_IDS = new int[] {
+ CU.cacheId("partitioned1"),
+ CU.cacheId("partitioned2"),
+ CU.cacheId("partitioned3"),
+ CU.cacheId("partitioned4"),
+ CU.cacheId("replicated1"),
+ CU.cacheId("replicated2"),
+ CU.cacheId("replicated3"),
+ CU.cacheId("replicated4"),
+ };
+
+ /** */
+ private static final int PARTS = 1024;
+
+ /** */
+ private static final int PAGES = 10240;
+
+ /**
+ *
+ */
+ public void testDistributions() {
+ printPageIdDistribution(
+ CU.cacheId("partitioned"), 1024, 30_000, 32, 2.5f);
+
+ printPageIdDistribution(
+ CU.cacheId("partitioned"), 1024, 30_000, 64, 2.5f);
+
+ printPageIdDistribution(
+ CU.cacheId(null), 1024, 30_000, 32, 2.5f);
+ }
+
+ /**
+ * @param cacheId Cache id.
+ * @param parts Parts.
+ * @param pagesPerPartition Pages per partition.
+ * @param segments Segments.
+ * @param capFactor Capacity factor.
+ */
+ private void printPageIdDistribution(
+ int cacheId,
+ int parts,
+ int pagesPerPartition,
+ int segments,
+ float capFactor
+ ) {
+ int allIds = parts * pagesPerPartition;
+
+ int perSegmentSize = allIds / segments;
+ int capacity = (int)(perSegmentSize * capFactor);
+
+ info("Total ids: " + allIds);
+
+ List<Map<Integer, Integer>> collisionsPerSegment = new ArrayList<>(segments);
+
+ for (int i = 0; i < segments; i++)
+ collisionsPerSegment.add(new HashMap<Integer, Integer>(allIds / segments, 1.0f));
+
+ int[] numInSegment = new int[segments];
+
+ for (int p = 0; p < parts; p++) {
+ for (int i = 0; i < pagesPerPartition; i++) {
+ long pageId = PageIdUtils.pageId(p, (byte)0, i);
+
+ int segment = PageMemoryImpl.segmentIndex(cacheId, pageId, segments);
+
+ int idxInSegment = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
+
+ Map<Integer, Integer> idxCollisions = collisionsPerSegment.get(segment);
+
+ Integer old = idxCollisions.get(idxInSegment);
+ idxCollisions.put(idxInSegment, old == null ? 1 : old + 1);
+
+ numInSegment[segment]++;
+ }
+ }
+
+ for (int i = 0; i < collisionsPerSegment.size(); i++) {
+ Map<Integer, Integer> idxCollisions = collisionsPerSegment.get(i);
+
+ int distinctPositions = idxCollisions.size();
+
+ int totalCnt = 0;
+ int nonZero = 0;
+
+ for (Map.Entry<Integer, Integer> collision : idxCollisions.entrySet()) {
+ if (collision.getValue() != null) {
+ totalCnt += collision.getValue();
+ nonZero++;
+ }
+ }
+
+ info(String.format("Segment stats [i=%d, total=%d, distinct=%d, spaceUsed=%d%%, avgItCnt=%.1f + ']",
+ i, numInSegment[i], distinctPositions, distinctPositions * 100 / numInSegment[i],
+ (float)totalCnt / nonZero));
+ }
+
+ info("==========================================================");
+ }
+
+ /**
+ * Uncomment and run this test manually to get data to plot histogram for per-element distance from ideal.
+ * You can use Octave to plot the histogram:
+ * <pre>
+ * all = csvread("histo.txt");
+ * hist(all, 200)
+ * </pre>
+ *
+ * @throws Exception If failed.
+ */
+ public void _testRealHistory() throws Exception {
+ int capacity = CACHE_IDS.length * PARTS * PAGES;
+
+ info("Capacity: " + capacity);
+
+ long mem = FullPageIdTable.requiredMemory(capacity);
+
+ info(U.readableSize(mem, true));
+
+ UnsafeMemoryProvider prov = new UnsafeMemoryProvider(new JavaLogger());
+
+ prov.initialize(new long[] {mem});
+
+ DirectMemoryRegion region = prov.nextRegion();
+
+ try {
+ long seed = U.currentTimeMillis();
+
+ info("Seed: " + seed + "L; //");
+
+ Random rnd = new Random(seed);
+
+ FullPageIdTable tbl = new FullPageIdTable(region.address(), region.size(), true);
+
+ Map<T2<Integer, Integer>, Integer> allocated = new HashMap<>();
+
+ for (int i = 0; i < capacity; i++) {
+ int cacheId = CACHE_IDS[rnd.nextInt(CACHE_IDS.length)];
+ int partId = rnd.nextInt(PARTS);
+
+ T2<Integer, Integer> key = new T2<>(cacheId, partId);
+
+ Integer pageIdx = allocated.get(key);
+
+ pageIdx = pageIdx == null ? 1 : pageIdx + 1;
+
+ if (pageIdx > PAGES)
+ continue;
+
+ tbl.put(cacheId, PageIdUtils.pageId(partId, (byte)0, pageIdx), 1, 0);
+
+ allocated.put(key, pageIdx);
+
+ if (i > 0 && i % 100_000 == 0)
+ info("Done: " + i);
+ }
+
+ int[] scans = new int[capacity];
+
+ int cur = 0;
+
+ for (T2<Integer, Integer> key : allocated.keySet()) {
+ Integer alloc = allocated.get(key);
+
+ if (alloc != null) {
+ for (int idx = 1; idx <= alloc; idx++) {
+ scans[cur] = tbl.distanceFromIdeal(key.get1(), PageIdUtils.pageId(key.get2(), (byte)0, idx), 0);
+
+ assert scans[cur] != -1;
+
+ cur++;
+ }
+ }
+ }
+
+ try (FileOutputStream out = new FileOutputStream("histo.txt")) {
+ PrintWriter w = new PrintWriter(new OutputStreamWriter(out));
+
+ for (int scan : scans) {
+ if (scan != 0)
+ w.println(scan);
+ }
+
+ w.flush();
+ }
+ }
+ finally {
+ prov.shutdown();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
new file mode 100644
index 0000000..1fff1f0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.impl.PageMemoryNoLoadSelfTest;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
+import org.apache.ignite.internal.util.lang.GridInClosure3X;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.util.typedef.CIX3;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.GridTestKernalContext;
+
+/**
+ *
+ */
+public class PageMemoryImplNoLoadTest extends PageMemoryNoLoadSelfTest {
+ /**
+ * @return Page memory implementation.
+ */
+ @Override protected PageMemory memory() throws Exception {
+ File memDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "pagemem", false);
+
+ long[] sizes = new long[10];
+
+ for (int i = 0; i < sizes.length; i++)
+ sizes[i] = 5 * 1024 * 1024;
+
+ DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), memDir);
+
+ GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
+ new GridTestKernalContext(log),
+ null,
+ null,
+ null,
+ new NoOpPageStoreManager(),
+ new NoOpWALManager(),
+ new IgniteCacheDatabaseSharedManager(),
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null
+ );
+
+ return new PageMemoryImpl(
+ provider,
+ sizes,
+ sharedCtx,
+ PAGE_SIZE,
+ new CIX3<FullPageId, ByteBuffer, Integer>() {
+ @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuffer, Integer tag) {
+ assert false : "No evictions should happen during the test";
+ }
+ },
+ new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
+ @Override public void applyx(Long page, FullPageId fullId, PageMemoryEx pageMem) {
+ }
+ },
+ new CheckpointLockStateChecker() {
+ @Override public boolean checkpointLockIsHeldByThread() {
+ return true;
+ }
+ },
+ new MemoryMetricsImpl(new MemoryPolicyConfiguration()));
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testPageHandleDeallocation() throws Exception {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
new file mode 100644
index 0000000..0366eca
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
+import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
+import org.apache.ignite.internal.util.lang.GridInClosure3X;
+import org.apache.ignite.internal.util.typedef.CIX3;
+import org.apache.ignite.testframework.junits.GridTestKernalContext;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.testframework.junits.logger.GridTestLog4jLogger;
+
+/**
+ *
+ */
+public class PageMemoryImplTest extends GridCommonAbstractTest {
+ /** Mb. */
+ private static final long MB = 1024 * 1024;
+
+ /** Page size. */
+ private static final int PAGE_SIZE = 1024;
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testThatAllocationTooMuchPagesCauseToOOMException() throws Exception {
+ PageMemoryImpl memory = createPageMemory();
+
+ try {
+ while (!Thread.currentThread().isInterrupted())
+ memory.allocatePage(1, PageIdAllocator.INDEX_PARTITION, PageIdAllocator.FLAG_IDX);
+ }
+ catch (IgniteOutOfMemoryException ignore) {
+ //Success
+ }
+
+ assertFalse(memory.safeToUpdate());
+ }
+
+ /**
+ *
+ */
+ private PageMemoryImpl createPageMemory() throws Exception {
+ long[] sizes = new long[5];
+
+ for (int i = 0; i < sizes.length; i++)
+ sizes[i] = 1024 * MB / 4;
+
+ sizes[4] = 10 * MB;
+
+ DirectMemoryProvider provider = new UnsafeMemoryProvider(log);
+
+ GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
+ new GridTestKernalContext(new GridTestLog4jLogger()),
+ null,
+ null,
+ null,
+ new NoOpPageStoreManager(),
+ new NoOpWALManager(),
+ new IgniteCacheDatabaseSharedManager(),
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null
+ );
+
+ PageMemoryImpl mem = new PageMemoryImpl(
+ provider,
+ sizes,
+ sharedCtx,
+ PAGE_SIZE,
+ new CIX3<FullPageId, ByteBuffer, Integer>() {
+ @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuf, Integer tag) {
+ assert false : "No evictions should happen during the test";
+ }
+ },
+ new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
+ @Override public void applyx(Long page, FullPageId fullId, PageMemoryEx pageMem) {
+ }
+ }, new CheckpointLockStateChecker() {
+ @Override public boolean checkpointLockIsHeldByThread() {
+ return true;
+ }
+ },
+ new MemoryMetricsImpl(new MemoryPolicyConfiguration()));
+
+ mem.start();
+
+ return mem;
+ }
+}