You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2017/03/24 14:12:38 UTC
[48/50] [abbrv] ignite git commit: Merge branch master ignite-2.0 to
ignite-3477
http://git-wip-us.apache.org/repos/asf/ignite/blob/81ae2d83/leak.patch
----------------------------------------------------------------------
diff --cc leak.patch
index 0000000,0000000..f618073
new file mode 100644
--- /dev/null
+++ b/leak.patch
@@@ -1,0 -1,0 +1,10521 @@@
++From 3c9e3786a21ec7c8919de9c29bc04d2e3561846c Mon Sep 17 00:00:00 2001
++From: Igor Seliverstov <gv...@gmail.com>
++Date: Wed, 15 Feb 2017 13:41:08 +0300
++Subject: [PATCH 01/41] IGNITE-4694 Add tests to check there are no memory
++ leaks in PageMemory
++
++---
++ .../processors/database/IgniteDbAbstractTest.java | 360 +++++++++++++++++++++
++ .../database/IgniteDbMemoryLeakAbstractTest.java | 84 +++++
++ .../database/IgniteDbMemoryLeakIndexedTest.java | 85 +++++
++ .../IgniteDbMemoryLeakLargeObjectsTest.java | 95 ++++++
++ .../database/IgniteDbMemoryLeakLargePagesTest.java | 90 ++++++
++ .../database/IgniteDbMemoryLeakTest.java | 85 +++++
++ .../IgniteDbMemoryLeakWithExpirationTest.java | 92 ++++++
++ .../database/IgniteDbPutGetAbstractTest.java | 347 +-------------------
++ 8 files changed, 903 insertions(+), 335 deletions(-)
++ create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
++ create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
++ create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
++ create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
++ create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
++ create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
++ create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
++
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
++new file mode 100644
++index 0000000..3bc7004
++--- /dev/null
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
++@@ -0,0 +1,360 @@
+++/*
+++ * Licensed to the Apache Software Foundation (ASF) under one or more
+++ * contributor license agreements. See the NOTICE file distributed with
+++ * this work for additional information regarding copyright ownership.
+++ * The ASF licenses this file to You under the Apache License, Version 2.0
+++ * (the "License"); you may not use this file except in compliance with
+++ * the License. You may obtain a copy of the License at
+++ *
+++ * http://www.apache.org/licenses/LICENSE-2.0
+++ *
+++ * Unless required by applicable law or agreed to in writing, software
+++ * distributed under the License is distributed on an "AS IS" BASIS,
+++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+++ * See the License for the specific language governing permissions and
+++ * limitations under the License.
+++ */
+++
+++package org.apache.ignite.internal.processors.database;
+++
+++import org.apache.ignite.cache.CacheAtomicityMode;
+++import org.apache.ignite.cache.CacheRebalanceMode;
+++import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+++import org.apache.ignite.cache.affinity.AffinityFunction;
+++import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+++import org.apache.ignite.cache.query.annotations.QuerySqlField;
+++import org.apache.ignite.configuration.CacheConfiguration;
+++import org.apache.ignite.configuration.IgniteConfiguration;
+++import org.apache.ignite.configuration.MemoryConfiguration;
+++import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
+++import org.apache.ignite.internal.util.typedef.internal.S;
+++import org.apache.ignite.internal.util.typedef.internal.U;
+++import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+++import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+++import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+++import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+++
+++import java.io.Serializable;
+++import java.util.Arrays;
+++import java.util.Random;
+++
+++/**
+++ *
+++ */
+++public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
+++ /** */
+++ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+++
+++ /**
+++ * @return Node count.
+++ */
+++ protected abstract int gridCount();
+++
+++ /**
+++ * @return {@code True} if indexing is enabled.
+++ */
+++ protected abstract boolean indexingEnabled();
+++
+++ /** {@inheritDoc} */
+++ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+++ IgniteConfiguration cfg = super.getConfiguration(gridName);
+++
+++ MemoryConfiguration dbCfg = new MemoryConfiguration();
+++
+++ dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
+++
+++ if (isLargePage())
+++ dbCfg.setPageSize(16 * 1024);
+++ else
+++ dbCfg.setPageSize(1024);
+++
+++
+++ dbCfg.setPageCacheSize(200 * 1024 * 1024);
+++
+++ configure(dbCfg);
+++
+++ cfg.setMemoryConfiguration(dbCfg);
+++
+++ CacheConfiguration ccfg = new CacheConfiguration();
+++
+++ if (indexingEnabled())
+++ ccfg.setIndexedTypes(Integer.class, DbValue.class);
+++
+++ ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+++ ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+++ ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
+++ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+++
+++ CacheConfiguration ccfg2 = new CacheConfiguration("non-primitive");
+++
+++ if (indexingEnabled())
+++ ccfg2.setIndexedTypes(DbKey.class, DbValue.class);
+++
+++ ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+++ ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+++ ccfg2.setRebalanceMode(CacheRebalanceMode.SYNC);
+++ ccfg2.setAffinity(new RendezvousAffinityFunction(false, 32));
+++
+++ CacheConfiguration ccfg3 = new CacheConfiguration("large");
+++
+++ if (indexingEnabled())
+++ ccfg3.setIndexedTypes(Integer.class, LargeDbValue.class);
+++
+++ ccfg3.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+++ ccfg3.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+++ ccfg3.setRebalanceMode(CacheRebalanceMode.SYNC);
+++ ccfg3.setAffinity(new RendezvousAffinityFunction(false, 32));
+++
+++ CacheConfiguration ccfg4 = new CacheConfiguration("tiny");
+++
+++ ccfg4.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+++ ccfg4.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+++ ccfg4.setRebalanceMode(CacheRebalanceMode.SYNC);
+++ ccfg4.setAffinity(new RendezvousAffinityFunction(false, 32));
+++
+++ final AffinityFunction aff = new RendezvousAffinityFunction(1, null);
+++
+++ ccfg4.setAffinity(aff);
+++
+++ cfg.setCacheConfiguration(ccfg, ccfg2, ccfg3, ccfg4);
+++
+++ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+++
+++ discoSpi.setIpFinder(IP_FINDER);
+++
+++ cfg.setDiscoverySpi(discoSpi);
+++ cfg.setMarshaller(null);
+++
+++ configure(cfg);
+++
+++ return cfg;
+++ }
+++
+++ protected void configure(IgniteConfiguration cfg){
+++ //NOP
+++ }
+++
+++ protected void configure(MemoryConfiguration mCfg){
+++ //NOP
+++ }
+++
+++ /** {@inheritDoc} */
+++ @Override protected void beforeTest() throws Exception {
+++ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+++
+++ long seed = 1464583813940L; // System.currentTimeMillis();
+++
+++ info("Seed: " + seed + "L");
+++
+++ BPlusTree.rnd = new Random(seed);
+++
+++ startGrids(gridCount());
+++
+++ awaitPartitionMapExchange();
+++ }
+++
+++ /** {@inheritDoc} */
+++ @Override protected void afterTest() throws Exception {
+++ BPlusTree.rnd = null;
+++
+++ stopAllGrids();
+++
+++ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+++ }
+++
+++ /**
+++ * @return {@code True} if use large page.
+++ */
+++ protected boolean isLargePage() {
+++ return false;
+++ }
+++
+++ /**
+++ *
+++ */
+++ static class DbKey implements Serializable {
+++ /** */
+++ int val;
+++
+++ /**
+++ * @param val Value.
+++ */
+++ DbKey(int val) {
+++ this.val = val;
+++ }
+++
+++ /** {@inheritDoc} */
+++ @Override public boolean equals(Object o) {
+++ if (this == o)
+++ return true;
+++
+++ if (o == null || !(o instanceof DbKey))
+++ return false;
+++
+++ DbKey key = (DbKey)o;
+++
+++ return val == key.val;
+++ }
+++
+++ /** {@inheritDoc} */
+++ @Override public int hashCode() {
+++ return val;
+++ }
+++ }
+++
+++ /**
+++ *
+++ */
+++ static class LargeDbKey implements Serializable {
+++ /** */
+++ int val;
+++
+++ /** */
+++ byte[] data;
+++
+++ /**
+++ * @param val Value.
+++ * @param size Key payload size.
+++ */
+++ LargeDbKey(int val, int size) {
+++ this.val = val;
+++
+++ data = new byte[size];
+++
+++ Arrays.fill(data, (byte)val);
+++ }
+++
+++ /** {@inheritDoc} */
+++ @Override public boolean equals(Object o) {
+++ if (this == o)
+++ return true;
+++
+++ if (o == null || !(o instanceof LargeDbKey))
+++ return false;
+++
+++ LargeDbKey key = (LargeDbKey)o;
+++
+++ return val == key.val && Arrays.equals(data, key.data);
+++ }
+++
+++ /** {@inheritDoc} */
+++ @Override public int hashCode() {
+++ return val + Arrays.hashCode(data);
+++ }
+++ }
+++
+++ /**
+++ *
+++ */
+++ static class DbValue implements Serializable {
+++ /** */
+++ @QuerySqlField(index = true)
+++ int iVal;
+++
+++ /** */
+++ @QuerySqlField(index = true)
+++ String sVal;
+++
+++ /** */
+++ @QuerySqlField
+++ long lVal;
+++
+++
+++
+++ /**
+++ * @param iVal Integer value.
+++ * @param sVal String value.
+++ * @param lVal Long value.
+++ */
+++ DbValue(int iVal, String sVal, long lVal) {
+++ this.iVal = iVal;
+++ this.sVal = sVal;
+++ this.lVal = lVal;
+++ }
+++
+++ /** {@inheritDoc} */
+++ @Override public boolean equals(Object o) {
+++ if (this == o)
+++ return true;
+++
+++ if (o == null || getClass() != o.getClass())
+++ return false;
+++
+++ DbValue dbVal = (DbValue)o;
+++
+++ return iVal == dbVal.iVal && lVal == dbVal.lVal &&
+++ !(sVal != null ? !sVal.equals(dbVal.sVal) : dbVal.sVal != null);
+++ }
+++
+++ /** {@inheritDoc} */
+++ @Override public int hashCode() {
+++ int res = iVal;
+++
+++ res = 31 * res + (sVal != null ? sVal.hashCode() : 0);
+++ res = 31 * res + (int)(lVal ^ (lVal >>> 32));
+++
+++ return res;
+++ }
+++
+++ /** {@inheritDoc} */
+++ @Override public String toString() {
+++ return S.toString(DbValue.class, this);
+++ }
+++ }
+++
+++ /**
+++ *
+++ */
+++ static class LargeDbValue {
+++ /** */
+++ @QuerySqlField(index = true)
+++ String str1;
+++
+++ /** */
+++ @QuerySqlField(index = true)
+++ String str2;
+++
+++ /** */
+++ int[] arr;
+++
+++ /**
+++ * @param str1 String 1.
+++ * @param str2 String 2.
+++ * @param arr Big array.
+++ */
+++ LargeDbValue(final String str1, final String str2, final int[] arr) {
+++ this.str1 = str1;
+++ this.str2 = str2;
+++ this.arr = arr;
+++ }
+++
+++ /** {@inheritDoc} */
+++ @Override public boolean equals(final Object o) {
+++ if (this == o) return true;
+++ if (o == null || getClass() != o.getClass()) return false;
+++
+++ final LargeDbValue that = (LargeDbValue) o;
+++
+++ if (str1 != null ? !str1.equals(that.str1) : that.str1 != null) return false;
+++ if (str2 != null ? !str2.equals(that.str2) : that.str2 != null) return false;
+++
+++ return Arrays.equals(arr, that.arr);
+++
+++ }
+++
+++ /** {@inheritDoc} */
+++ @Override public int hashCode() {
+++ int res = str1 != null ? str1.hashCode() : 0;
+++
+++ res = 31 * res + (str2 != null ? str2.hashCode() : 0);
+++ res = 31 * res + Arrays.hashCode(arr);
+++
+++ return res;
+++ }
+++
+++ /** {@inheritDoc} */
+++ @Override public String toString() {
+++ return S.toString(LargeDbValue.class, this);
+++ }
+++ }
+++}
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
++new file mode 100644
++index 0000000..6a5d039
++--- /dev/null
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
++@@ -0,0 +1,84 @@
+++/*
+++ * Licensed to the Apache Software Foundation (ASF) under one or more
+++ * contributor license agreements. See the NOTICE file distributed with
+++ * this work for additional information regarding copyright ownership.
+++ * The ASF licenses this file to You under the Apache License, Version 2.0
+++ * (the "License"); you may not use this file except in compliance with
+++ * the License. You may obtain a copy of the License at
+++ *
+++ * http://www.apache.org/licenses/LICENSE-2.0
+++ *
+++ * Unless required by applicable law or agreed to in writing, software
+++ * distributed under the License is distributed on an "AS IS" BASIS,
+++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+++ * See the License for the specific language governing permissions and
+++ * limitations under the License.
+++ */
+++
+++package org.apache.ignite.internal.processors.database;
+++
+++import org.apache.ignite.Ignite;
+++import org.apache.ignite.IgniteCompute;
+++import org.apache.ignite.compute.ComputeTaskFuture;
+++import org.apache.ignite.internal.IgniteEx;
+++import org.apache.ignite.lang.IgniteRunnable;
+++import org.apache.ignite.resources.IgniteInstanceResource;
+++
+++import java.util.concurrent.TimeUnit;
+++
+++/**
+++ *
+++ */
+++public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest {
+++
+++ /** Test duration in seconds*/
+++ protected abstract int duration();
+++
+++ @Override
+++ protected long getTestTimeout() {
+++ return duration() * 1200;
+++ }
+++
+++ /** */
+++ protected abstract void operation(IgniteEx ig);
+++
+++ /** */
+++ public void testMemoryLeak() throws Exception {
+++
+++ final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration());
+++
+++ int tasksCount = Runtime.getRuntime().availableProcessors() * 4;
+++
+++ IgniteCompute compute = grid(0).compute().withAsync();
+++
+++ ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCount];
+++
+++ for (int i = 0; i < tasksCount; i++) {
+++ compute.run(new IgniteRunnable() {
+++ @IgniteInstanceResource
+++ private Ignite ig;
+++
+++ @Override
+++ public void run() {
+++ int i = 0;
+++ while (System.nanoTime() < end) {
+++ operation((IgniteEx) ig);
+++
+++ if(i++ == 100) {
+++ check((IgniteEx) ig);
+++ i = 0;
+++ }
+++ }
+++ }
+++ });
+++
+++ futs[i] = compute.future();
+++ }
+++
+++ for (ComputeTaskFuture fut : futs) {
+++ fut.get();
+++ }
+++ }
+++
+++ protected void check(IgniteEx ig) {}
+++}
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
++new file mode 100644
++index 0000000..4cd74d0
++--- /dev/null
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
++@@ -0,0 +1,85 @@
+++/*
+++ * Licensed to the Apache Software Foundation (ASF) under one or more
+++ * contributor license agreements. See the NOTICE file distributed with
+++ * this work for additional information regarding copyright ownership.
+++ * The ASF licenses this file to You under the Apache License, Version 2.0
+++ * (the "License"); you may not use this file except in compliance with
+++ * the License. You may obtain a copy of the License at
+++ *
+++ * http://www.apache.org/licenses/LICENSE-2.0
+++ *
+++ * Unless required by applicable law or agreed to in writing, software
+++ * distributed under the License is distributed on an "AS IS" BASIS,
+++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+++ * See the License for the specific language governing permissions and
+++ * limitations under the License.
+++ */
+++
+++package org.apache.ignite.internal.processors.database;
+++
+++import org.apache.ignite.IgniteCache;
+++import org.apache.ignite.configuration.IgniteConfiguration;
+++import org.apache.ignite.configuration.MemoryConfiguration;
+++import org.apache.ignite.internal.IgniteEx;
+++
+++import java.util.Random;
+++import java.util.concurrent.ThreadLocalRandom;
+++
+++/**
+++ *
+++ */
+++public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakAbstractTest {
+++
+++ @Override
+++ protected int duration() {
+++ return 300;
+++ }
+++
+++ @Override
+++ protected int gridCount() {
+++ return 1;
+++ }
+++
+++ @Override
+++ protected void configure(IgniteConfiguration cfg) {
+++ cfg.setMetricsLogFrequency(5000);
+++ }
+++
+++ @Override
+++ protected void configure(MemoryConfiguration mCfg) {
+++ mCfg.setPageCacheSize(1024 * 1024);
+++ }
+++
+++ @Override
+++ protected boolean indexingEnabled() {
+++ return true;
+++ }
+++
+++ protected void operation(IgniteEx ig){
+++ IgniteCache<Object, Object> cache = ig.cache("non-primitive");
+++ Random rnd = ThreadLocalRandom.current();
+++
+++ for (int i = 0; i < 1000; i++) {
+++ DbKey key = new DbKey(rnd.nextInt(200_000));
+++
+++ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
+++
+++ switch (rnd.nextInt(3)) {
+++ case 0:
+++ cache.getAndPut(key, v0);
+++ case 1:
+++ cache.get(key);
+++ break;
+++ case 2:
+++ cache.getAndRemove(key);
+++ }
+++ }
+++ }
+++
+++ @Override
+++ protected void check(IgniteEx ig) {
+++ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+++
+++ assertTrue(pages < 19100);
+++ }
+++}
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
++new file mode 100644
++index 0000000..a4d88e1
++--- /dev/null
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
++@@ -0,0 +1,95 @@
+++/*
+++ * Licensed to the Apache Software Foundation (ASF) under one or more
+++ * contributor license agreements. See the NOTICE file distributed with
+++ * this work for additional information regarding copyright ownership.
+++ * The ASF licenses this file to You under the Apache License, Version 2.0
+++ * (the "License"); you may not use this file except in compliance with
+++ * the License. You may obtain a copy of the License at
+++ *
+++ * http://www.apache.org/licenses/LICENSE-2.0
+++ *
+++ * Unless required by applicable law or agreed to in writing, software
+++ * distributed under the License is distributed on an "AS IS" BASIS,
+++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+++ * See the License for the specific language governing permissions and
+++ * limitations under the License.
+++ */
+++
+++package org.apache.ignite.internal.processors.database;
+++
+++import org.apache.ignite.IgniteCache;
+++import org.apache.ignite.configuration.IgniteConfiguration;
+++import org.apache.ignite.configuration.MemoryConfiguration;
+++import org.apache.ignite.internal.IgniteEx;
+++
+++import java.util.Random;
+++import java.util.concurrent.ThreadLocalRandom;
+++
+++/**
+++ *
+++ */
+++public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstractTest {
+++
+++ private final static int[] ARRAY;
+++ static {
+++ ARRAY = new int[1024];
+++ Random rnd = new Random();
+++ for (int i = 0; i < ARRAY.length; i++) {
+++ ARRAY[i] = rnd.nextInt();
+++ }
+++
+++ }
+++
+++ @Override
+++ protected int duration() {
+++ return 300;
+++ }
+++
+++ @Override
+++ protected int gridCount() {
+++ return 1;
+++ }
+++
+++ @Override
+++ protected void configure(IgniteConfiguration cfg) {
+++ cfg.setMetricsLogFrequency(5000);
+++ }
+++
+++ @Override
+++ protected void configure(MemoryConfiguration mCfg) {
+++ mCfg.setPageCacheSize(60 * 1024 * 1024);
+++ }
+++
+++ @Override
+++ protected boolean indexingEnabled() {
+++ return false;
+++ }
+++
+++ protected void operation(IgniteEx ig){
+++ IgniteCache<Object, Object> cache = ig.cache("large");
+++ Random rnd = ThreadLocalRandom.current();
+++
+++ for (int i = 0; i < 1000; i++) {
+++ LargeDbKey key = new LargeDbKey(rnd.nextInt(10_000), 1024);
+++
+++ LargeDbValue v0 = new LargeDbValue("test-value-1-" + rnd.nextInt(200), "test-value-2-" + rnd.nextInt(200), ARRAY);
+++
+++ switch (rnd.nextInt(3)) {
+++ case 0:
+++ cache.getAndPut(key, v0);
+++ case 1:
+++ cache.get(key);
+++ break;
+++ case 2:
+++ cache.getAndRemove(key);
+++ }
+++ }
+++ }
+++
+++ @Override
+++ protected void check(IgniteEx ig) {
+++ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+++
+++ assertTrue(pages < 50000);
+++ }
+++}
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
++new file mode 100644
++index 0000000..bfa4aa9
++--- /dev/null
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
++@@ -0,0 +1,90 @@
+++/*
+++ * Licensed to the Apache Software Foundation (ASF) under one or more
+++ * contributor license agreements. See the NOTICE file distributed with
+++ * this work for additional information regarding copyright ownership.
+++ * The ASF licenses this file to You under the Apache License, Version 2.0
+++ * (the "License"); you may not use this file except in compliance with
+++ * the License. You may obtain a copy of the License at
+++ *
+++ * http://www.apache.org/licenses/LICENSE-2.0
+++ *
+++ * Unless required by applicable law or agreed to in writing, software
+++ * distributed under the License is distributed on an "AS IS" BASIS,
+++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+++ * See the License for the specific language governing permissions and
+++ * limitations under the License.
+++ */
+++
+++package org.apache.ignite.internal.processors.database;
+++
+++import org.apache.ignite.IgniteCache;
+++import org.apache.ignite.configuration.IgniteConfiguration;
+++import org.apache.ignite.configuration.MemoryConfiguration;
+++import org.apache.ignite.internal.IgniteEx;
+++
+++import java.util.Random;
+++import java.util.concurrent.ThreadLocalRandom;
+++
+++/**
+++ *
+++ */
+++public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakAbstractTest {
+++
+++ @Override
+++ protected int duration() {
+++ return 300;
+++ }
+++
+++ @Override
+++ protected int gridCount() {
+++ return 1;
+++ }
+++
+++ @Override
+++ protected void configure(IgniteConfiguration cfg) {
+++ cfg.setMetricsLogFrequency(5000);
+++ }
+++
+++ @Override
+++ protected void configure(MemoryConfiguration mCfg) {
+++ mCfg.setPageCacheSize(100 * 1024 * 1024);
+++ }
+++
+++ @Override
+++ protected boolean indexingEnabled() {
+++ return false;
+++ }
+++
+++ @Override
+++ protected boolean isLargePage() {
+++ return true;
+++ }
+++
+++ protected void operation(IgniteEx ig){
+++ IgniteCache<Object, Object> cache = ig.cache("non-primitive");
+++ Random rnd = ThreadLocalRandom.current();
+++
+++ for (int i = 0; i < 1000; i++) {
+++ DbKey key = new DbKey(rnd.nextInt(200_000));
+++
+++ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
+++
+++ switch (rnd.nextInt(3)) {
+++ case 0:
+++ cache.getAndPut(key, v0);
+++ case 1:
+++ cache.get(key);
+++ break;
+++ case 2:
+++ cache.getAndRemove(key);
+++ }
+++ }
+++ }
+++
+++ @Override
+++ protected void check(IgniteEx ig) {
+++ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+++
+++ assertTrue(pages < 4600);
+++ }
+++}
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
++new file mode 100644
++index 0000000..6af4e41
++--- /dev/null
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
++@@ -0,0 +1,85 @@
+++/*
+++ * Licensed to the Apache Software Foundation (ASF) under one or more
+++ * contributor license agreements. See the NOTICE file distributed with
+++ * this work for additional information regarding copyright ownership.
+++ * The ASF licenses this file to You under the Apache License, Version 2.0
+++ * (the "License"); you may not use this file except in compliance with
+++ * the License. You may obtain a copy of the License at
+++ *
+++ * http://www.apache.org/licenses/LICENSE-2.0
+++ *
+++ * Unless required by applicable law or agreed to in writing, software
+++ * distributed under the License is distributed on an "AS IS" BASIS,
+++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+++ * See the License for the specific language governing permissions and
+++ * limitations under the License.
+++ */
+++
+++package org.apache.ignite.internal.processors.database;
+++
+++import org.apache.ignite.IgniteCache;
+++import org.apache.ignite.configuration.IgniteConfiguration;
+++import org.apache.ignite.configuration.MemoryConfiguration;
+++import org.apache.ignite.internal.IgniteEx;
+++
+++import java.util.Random;
+++import java.util.concurrent.ThreadLocalRandom;
+++
+++/**
+++ *
+++ */
+++public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
+++
+++ @Override
+++ protected int duration() {
+++ return 300;
+++ }
+++
+++ @Override
+++ protected int gridCount() {
+++ return 1;
+++ }
+++
+++ @Override
+++ protected void configure(IgniteConfiguration cfg) {
+++ cfg.setMetricsLogFrequency(5000);
+++ }
+++
+++ @Override
+++ protected void configure(MemoryConfiguration mCfg) {
+++ mCfg.setPageCacheSize(1024 * 1024);
+++ }
+++
+++ @Override
+++ protected boolean indexingEnabled() {
+++ return false;
+++ }
+++
+++ protected void operation(IgniteEx ig){
+++ IgniteCache<Object, Object> cache = ig.cache("non-primitive");
+++ Random rnd = ThreadLocalRandom.current();
+++
+++ for (int i = 0; i < 1000; i++) {
+++ DbKey key = new DbKey(rnd.nextInt(200_000));
+++
+++ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
+++
+++ switch (rnd.nextInt(3)) {
+++ case 0:
+++ cache.getAndPut(key, v0);
+++ case 1:
+++ cache.get(key);
+++ break;
+++ case 2:
+++ cache.getAndRemove(key);
+++ }
+++ }
+++ }
+++
+++ @Override
+++ protected void check(IgniteEx ig) {
+++ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+++
+++ assertTrue(pages < 19100);
+++ }
+++}
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
++new file mode 100644
++index 0000000..d9e3f34
++--- /dev/null
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
++@@ -0,0 +1,92 @@
+++/*
+++ * Licensed to the Apache Software Foundation (ASF) under one or more
+++ * contributor license agreements. See the NOTICE file distributed with
+++ * this work for additional information regarding copyright ownership.
+++ * The ASF licenses this file to You under the Apache License, Version 2.0
+++ * (the "License"); you may not use this file except in compliance with
+++ * the License. You may obtain a copy of the License at
+++ *
+++ * http://www.apache.org/licenses/LICENSE-2.0
+++ *
+++ * Unless required by applicable law or agreed to in writing, software
+++ * distributed under the License is distributed on an "AS IS" BASIS,
+++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+++ * See the License for the specific language governing permissions and
+++ * limitations under the License.
+++ */
+++
+++package org.apache.ignite.internal.processors.database;
+++
+++import org.apache.ignite.IgniteCache;
+++import org.apache.ignite.configuration.IgniteConfiguration;
+++import org.apache.ignite.configuration.MemoryConfiguration;
+++import org.apache.ignite.internal.IgniteEx;
+++
+++import javax.cache.expiry.CreatedExpiryPolicy;
+++import javax.cache.expiry.Duration;
+++import javax.cache.expiry.ExpiryPolicy;
+++import java.util.Random;
+++import java.util.concurrent.ThreadLocalRandom;
+++
+++import static java.util.concurrent.TimeUnit.MILLISECONDS;
+++
+++/**
+++ *
+++ */
+++public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakAbstractTest {
+++
+++ private static final ExpiryPolicy EXPIRY = new CreatedExpiryPolicy(new Duration(MILLISECONDS, 10L));
+++
+++ @Override
+++ protected int duration() {
+++ return 300;
+++ }
+++
+++ @Override
+++ protected int gridCount() {
+++ return 1;
+++ }
+++
+++ @Override
+++ protected void configure(IgniteConfiguration cfg) {
+++ cfg.setMetricsLogFrequency(5000);
+++ }
+++
+++ @Override
+++ protected void configure(MemoryConfiguration mCfg) {
+++ mCfg.setPageCacheSize(1024 * 1024);
+++ }
+++
+++ @Override
+++ protected boolean indexingEnabled() {
+++ return false;
+++ }
+++
+++ protected void operation(IgniteEx ig) {
+++ IgniteCache<Object, Object> cache = ig.cache("non-primitive").withExpiryPolicy(EXPIRY);
+++ Random rnd = ThreadLocalRandom.current();
+++
+++ for (int i = 0; i < 1000; i++) {
+++ DbKey key = new DbKey(rnd.nextInt(200_000));
+++
+++ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
+++
+++ switch (rnd.nextInt(3)) {
+++ case 0:
+++ cache.getAndPut(key, v0);
+++ case 1:
+++ cache.get(key);
+++ break;
+++ case 2:
+++ cache.getAndRemove(key);
+++ }
+++ }
+++ }
+++
+++ @Override
+++ protected void check(IgniteEx ig) {
+++ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+++
+++ assertTrue(pages < 10000);
+++ }
+++}
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
++index c7a07e3..228a262 100644
++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
++@@ -17,175 +17,39 @@
++
++ package org.apache.ignite.internal.processors.database;
++
++-import java.io.Serializable;
++-import java.util.Arrays;
++-import java.util.HashMap;
++-import java.util.HashSet;
++-import java.util.LinkedHashMap;
++-import java.util.List;
++-import java.util.Map;
++-import java.util.Random;
++-import java.util.Set;
++-import java.util.UUID;
++-import java.util.concurrent.ThreadLocalRandom;
++-import javax.cache.Cache;
++ import org.apache.ignite.Ignite;
++ import org.apache.ignite.IgniteCache;
++ import org.apache.ignite.IgniteDataStreamer;
++-import org.apache.ignite.cache.CacheAtomicityMode;
++ import org.apache.ignite.cache.CachePeekMode;
++-import org.apache.ignite.cache.CacheRebalanceMode;
++-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
++ import org.apache.ignite.cache.affinity.Affinity;
++-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
++-import org.apache.ignite.cache.affinity.AffinityFunction;
++ import org.apache.ignite.cache.query.QueryCursor;
++ import org.apache.ignite.cache.query.ScanQuery;
++ import org.apache.ignite.cache.query.SqlFieldsQuery;
++ import org.apache.ignite.cache.query.SqlQuery;
++-import org.apache.ignite.cache.query.annotations.QuerySqlField;
++-import org.apache.ignite.configuration.CacheConfiguration;
++-import org.apache.ignite.configuration.MemoryConfiguration;
++-import org.apache.ignite.configuration.IgniteConfiguration;
++ import org.apache.ignite.internal.IgniteEx;
++ import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
++ import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
++ import org.apache.ignite.internal.util.GridRandom;
++ import org.apache.ignite.internal.util.typedef.PA;
++ import org.apache.ignite.internal.util.typedef.X;
++-import org.apache.ignite.internal.util.typedef.internal.S;
++-import org.apache.ignite.internal.util.typedef.internal.U;
++-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
++-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
++-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
++ import org.apache.ignite.testframework.GridTestUtils;
++-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
++ import org.junit.Assert;
++
+++import javax.cache.Cache;
+++import java.util.HashMap;
+++import java.util.HashSet;
+++import java.util.LinkedHashMap;
+++import java.util.List;
+++import java.util.Map;
+++import java.util.Random;
+++import java.util.Set;
+++import java.util.UUID;
+++import java.util.concurrent.ThreadLocalRandom;
+++
++ /**
++ *
++ */
++-public abstract class IgniteDbPutGetAbstractTest extends GridCommonAbstractTest {
++- /** */
++- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
++-
++- /**
++- * @return Node count.
++- */
++- protected abstract int gridCount();
++-
++- /**
++- * @return {@code True} if indexing is enabled.
++- */
++- protected abstract boolean indexingEnabled();
++-
++- /** {@inheritDoc} */
++- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
++- IgniteConfiguration cfg = super.getConfiguration(gridName);
++-
++- MemoryConfiguration dbCfg = new MemoryConfiguration();
++-
++- if (isLargePage()) {
++- dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
++-
++- dbCfg.setPageSize(16 * 1024);
++-
++- dbCfg.setPageCacheSize(200 * 1024 * 1024);
++- }
++- else {
++- dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
++-
++- dbCfg.setPageSize(1024);
++-
++- dbCfg.setPageCacheSize(200 * 1024 * 1024);
++- }
++-
++- cfg.setMemoryConfiguration(dbCfg);
++-
++- CacheConfiguration ccfg = new CacheConfiguration();
++-
++- if (indexingEnabled())
++- ccfg.setIndexedTypes(Integer.class, DbValue.class);
++-
++- ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
++- ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
++- ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
++- ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
++-
++- CacheConfiguration ccfg2 = new CacheConfiguration("non-primitive");
++-
++- if (indexingEnabled())
++- ccfg2.setIndexedTypes(DbKey.class, DbValue.class);
++-
++- ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
++- ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
++- ccfg2.setRebalanceMode(CacheRebalanceMode.SYNC);
++- ccfg2.setAffinity(new RendezvousAffinityFunction(false, 32));
++-
++- CacheConfiguration ccfg3 = new CacheConfiguration("large");
++-
++- if (indexingEnabled())
++- ccfg3.setIndexedTypes(Integer.class, LargeDbValue.class);
++-
++- ccfg3.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
++- ccfg3.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
++- ccfg3.setRebalanceMode(CacheRebalanceMode.SYNC);
++- ccfg3.setAffinity(new RendezvousAffinityFunction(false, 32));
++-
++- CacheConfiguration ccfg4 = new CacheConfiguration("tiny");
++-
++- ccfg4.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
++- ccfg4.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
++- ccfg4.setRebalanceMode(CacheRebalanceMode.SYNC);
++- ccfg4.setAffinity(new RendezvousAffinityFunction(false, 32));
++-
++- final AffinityFunction aff = new RendezvousAffinityFunction(1, null);
++-
++- ccfg4.setAffinity(aff);
++-
++- cfg.setCacheConfiguration(ccfg, ccfg2, ccfg3, ccfg4);
++-
++- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
++-
++- discoSpi.setIpFinder(IP_FINDER);
++-
++- cfg.setDiscoverySpi(discoSpi);
++- cfg.setMarshaller(null);
++-
++- return cfg;
++- }
++-
++- /** {@inheritDoc} */
++- @Override protected void beforeTest() throws Exception {
++- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
++-
++- long seed = 1464583813940L; // System.currentTimeMillis();
++-
++- info("Seed: " + seed + "L");
++-
++- BPlusTree.rnd = new Random(seed);
++-
++- startGrids(gridCount());
++-
++- awaitPartitionMapExchange();
++- }
++-
++- /** {@inheritDoc} */
++- @Override protected void afterTest() throws Exception {
++- BPlusTree.rnd = null;
++-
++- stopAllGrids();
++-
++- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
++- }
++-
++- /**
++- * @return {@code True} if use large page.
++- */
++- protected boolean isLargePage() {
++- return false;
++- };
++-
+++public abstract class IgniteDbPutGetAbstractTest extends IgniteDbAbstractTest {
++ /**
++ *
++ */
++@@ -1349,191 +1213,4 @@ private void checkEmpty(final GridCacheAdapter internalCache, final Object key)
++
++ assertNull(internalCache.peekEx(key));
++ }
++-
++- /**
++- *
++- */
++- private static class DbKey implements Serializable {
++- /** */
++- private int val;
++-
++- /**
++- * @param val Value.
++- */
++- private DbKey(int val) {
++- this.val = val;
++- }
++-
++- /** {@inheritDoc} */
++- @Override public boolean equals(Object o) {
++- if (this == o)
++- return true;
++-
++- if (o == null || !(o instanceof DbKey))
++- return false;
++-
++- DbKey key = (DbKey)o;
++-
++- return val == key.val;
++- }
++-
++- /** {@inheritDoc} */
++- @Override public int hashCode() {
++- return val;
++- }
++- }
++-
++- /**
++- *
++- */
++- private static class LargeDbKey implements Serializable {
++- /** */
++- private int val;
++-
++- /** */
++- private byte[] data;
++-
++- /**
++- * @param val Value.
++- * @param size Key payload size.
++- */
++- private LargeDbKey(int val, int size) {
++- this.val = val;
++-
++- data = new byte[size];
++-
++- Arrays.fill(data, (byte)val);
++- }
++-
++- /** {@inheritDoc} */
++- @Override public boolean equals(Object o) {
++- if (this == o)
++- return true;
++-
++- if (o == null || !(o instanceof LargeDbKey))
++- return false;
++-
++- LargeDbKey key = (LargeDbKey)o;
++-
++- return val == key.val && Arrays.equals(data, key.data);
++- }
++-
++- /** {@inheritDoc} */
++- @Override public int hashCode() {
++- return val + Arrays.hashCode(data);
++- }
++- }
++-
++- /**
++- *
++- */
++- private static class DbValue implements Serializable {
++- /** */
++- @QuerySqlField(index = true)
++- private int iVal;
++-
++- /** */
++- @QuerySqlField(index = true)
++- private String sVal;
++-
++- /** */
++- @QuerySqlField
++- private long lVal;
++-
++- /**
++- * @param iVal Integer value.
++- * @param sVal String value.
++- * @param lVal Long value.
++- */
++- public DbValue(int iVal, String sVal, long lVal) {
++- this.iVal = iVal;
++- this.sVal = sVal;
++- this.lVal = lVal;
++- }
++-
++- /** {@inheritDoc} */
++- @Override public boolean equals(Object o) {
++- if (this == o)
++- return true;
++-
++- if (o == null || getClass() != o.getClass())
++- return false;
++-
++- DbValue dbVal = (DbValue)o;
++-
++- return iVal == dbVal.iVal && lVal == dbVal.lVal &&
++- !(sVal != null ? !sVal.equals(dbVal.sVal) : dbVal.sVal != null);
++- }
++-
++- /** {@inheritDoc} */
++- @Override public int hashCode() {
++- int res = iVal;
++-
++- res = 31 * res + (sVal != null ? sVal.hashCode() : 0);
++- res = 31 * res + (int)(lVal ^ (lVal >>> 32));
++-
++- return res;
++- }
++-
++- /** {@inheritDoc} */
++- @Override public String toString() {
++- return S.toString(DbValue.class, this);
++- }
++- }
++-
++- /**
++- *
++- */
++- private static class LargeDbValue {
++- /** */
++- @QuerySqlField(index = true)
++- private String str1;
++-
++- /** */
++- @QuerySqlField(index = true)
++- private String str2;
++-
++- /** */
++- private int[] arr;
++-
++- /**
++- * @param str1 String 1.
++- * @param str2 String 2.
++- * @param arr Big array.
++- */
++- public LargeDbValue(final String str1, final String str2, final int[] arr) {
++- this.str1 = str1;
++- this.str2 = str2;
++- this.arr = arr;
++- }
++-
++- /** {@inheritDoc} */
++- @Override public boolean equals(final Object o) {
++- if (this == o) return true;
++- if (o == null || getClass() != o.getClass()) return false;
++-
++- final LargeDbValue that = (LargeDbValue) o;
++-
++- if (str1 != null ? !str1.equals(that.str1) : that.str1 != null) return false;
++- if (str2 != null ? !str2.equals(that.str2) : that.str2 != null) return false;
++-
++- return Arrays.equals(arr, that.arr);
++-
++- }
++-
++- /** {@inheritDoc} */
++- @Override public int hashCode() {
++- int res = str1 != null ? str1.hashCode() : 0;
++-
++- res = 31 * res + (str2 != null ? str2.hashCode() : 0);
++- res = 31 * res + Arrays.hashCode(arr);
++-
++- return res;
++- }
++-
++- /** {@inheritDoc} */
++- @Override public String toString() {
++- return S.toString(LargeDbValue.class, this);
++- }
++- }
++ }
++
++From 8e12097f9094d7f155135ee2f4c9c33f5f7af9aa Mon Sep 17 00:00:00 2001
++From: sboikov <sb...@gridgain.com>
++Date: Wed, 15 Feb 2017 15:08:14 +0300
++Subject: [PATCH 02/41] ignite-4694 review
++
++---
++ .../database/IgniteDbMemoryLeakAbstractTest.java | 15 ++++++++-------
++ .../database/IgniteDbMemoryLeakIndexedTest.java | 3 +++
++ .../database/IgniteDbMemoryLeakLargeObjectsTest.java | 9 +++++----
++ .../database/IgniteDbMemoryLeakLargePagesTest.java | 2 ++
++ .../processors/database/IgniteDbMemoryLeakTest.java | 10 +++++++---
++ .../database/IgniteDbMemoryLeakWithExpirationTest.java | 2 +-
++ 6 files changed, 26 insertions(+), 15 deletions(-)
++
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
++index 6a5d039..fc0e715 100644
++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
++@@ -27,10 +27,10 @@
++ import java.util.concurrent.TimeUnit;
++
++ /**
++- *
+++ * TODO: fix javadoc warnings, code style.
++ */
++ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest {
++-
+++ // TODO: take duration from system property.
++ /** Test duration in seconds*/
++ protected abstract int duration();
++
++@@ -44,16 +44,18 @@ protected long getTestTimeout() {
++
++ /** */
++ public void testMemoryLeak() throws Exception {
+++ // TODO: take PageMemory max size is the same as we configured.
++
++ final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration());
++
++- int tasksCount = Runtime.getRuntime().availableProcessors() * 4;
+++ // TODO: use threads instead of compute or make sure there are enough threads in pool.
+++ int tasksCnt = Runtime.getRuntime().availableProcessors() * 4;
++
++ IgniteCompute compute = grid(0).compute().withAsync();
++
++- ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCount];
+++ ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCnt];
++
++- for (int i = 0; i < tasksCount; i++) {
+++ for (int i = 0; i < tasksCnt; i++) {
++ compute.run(new IgniteRunnable() {
++ @IgniteInstanceResource
++ private Ignite ig;
++@@ -75,9 +77,8 @@ public void run() {
++ futs[i] = compute.future();
++ }
++
++- for (ComputeTaskFuture fut : futs) {
+++ for (ComputeTaskFuture fut : futs)
++ fut.get();
++- }
++ }
++
++ protected void check(IgniteEx ig) {}
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
++index 4cd74d0..db77131 100644
++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
++@@ -51,6 +51,7 @@ protected void configure(MemoryConfiguration mCfg) {
++ }
++
++ @Override
+++ // TODO: move test to module ignite-indexing.
++ protected boolean indexingEnabled() {
++ return true;
++ }
++@@ -64,6 +65,8 @@ protected void operation(IgniteEx ig){
++
++ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
++
+++ // TODO: also execute sql queries.
+++
++ switch (rnd.nextInt(3)) {
++ case 0:
++ cache.getAndPut(key, v0);
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
++index a4d88e1..2a6c8cd 100644
++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
++@@ -29,15 +29,16 @@
++ *
++ */
++ public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstractTest {
++-
+++ /** */
++ private final static int[] ARRAY;
+++
++ static {
++ ARRAY = new int[1024];
+++
++ Random rnd = new Random();
++- for (int i = 0; i < ARRAY.length; i++) {
++- ARRAY[i] = rnd.nextInt();
++- }
++
+++ for (int i = 0; i < ARRAY.length; i++)
+++ ARRAY[i] = rnd.nextInt();
++ }
++
++ @Override
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
++index bfa4aa9..91c96af 100644
++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
++@@ -47,6 +47,7 @@ protected void configure(IgniteConfiguration cfg) {
++
++ @Override
++ protected void configure(MemoryConfiguration mCfg) {
+++ // TODO: understand why such overhead with large pages.
++ mCfg.setPageCacheSize(100 * 1024 * 1024);
++ }
++
++@@ -60,6 +61,7 @@ protected boolean isLargePage() {
++ return true;
++ }
++
+++ // TODO: avoid copy/paste.
++ protected void operation(IgniteEx ig){
++ IgniteCache<Object, Object> cache = ig.cache("non-primitive");
++ Random rnd = ThreadLocalRandom.current();
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
++index 6af4e41..2b0ce1e 100644
++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
++@@ -29,9 +29,8 @@
++ *
++ */
++ public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
++-
++- @Override
++- protected int duration() {
+++ /** {@inheritDoc} */
+++ @Override protected int duration() {
++ return 300;
++ }
++
++@@ -64,12 +63,17 @@ protected void operation(IgniteEx ig){
++
++ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
++
+++ // TODO: also execute scan query.
+++
++ switch (rnd.nextInt(3)) {
++ case 0:
++ cache.getAndPut(key, v0);
+++ break;
+++
++ case 1:
++ cache.get(key);
++ break;
+++
++ case 2:
++ cache.getAndRemove(key);
++ }
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
++index d9e3f34..95fe8c8 100644
++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
++@@ -34,7 +34,7 @@
++ *
++ */
++ public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakAbstractTest {
++-
+++ /** */
++ private static final ExpiryPolicy EXPIRY = new CreatedExpiryPolicy(new Duration(MILLISECONDS, 10L));
++
++ @Override
++
++From e70d990f14288cfc8fe211fa25631016d5708144 Mon Sep 17 00:00:00 2001
++From: Igor Seliverstov <gv...@gmail.com>
++Date: Wed, 15 Feb 2017 18:04:38 +0300
++Subject: [PATCH 03/41] IGNITE-4694 Add tests to check there are no memory
++ leaks in PageMemory
++
++---
++ .../cache/IgniteCacheOffheapManagerImpl.java | 2 +-
++ .../processors/database/IgniteDbAbstractTest.java | 6 +
++ .../database/IgniteDbMemoryLeakAbstractTest.java | 172 ++++++++++++++++-----
++ .../database/IgniteDbMemoryLeakIndexedTest.java | 65 +-------
++ .../IgniteDbMemoryLeakLargeObjectsTest.java | 64 ++------
++ .../database/IgniteDbMemoryLeakLargePagesTest.java | 67 ++------
++ .../database/IgniteDbMemoryLeakTest.java | 63 ++------
++ .../IgniteDbMemoryLeakWithExpirationTest.java | 58 +------
++ 8 files changed, 182 insertions(+), 315 deletions(-)
++
++diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
++index 5df99b6..9becc99 100644
++--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
++@@ -897,7 +897,7 @@ public CacheDataStoreImpl(
++ */
++ private boolean canUpdateOldRow(@Nullable CacheDataRow oldRow, DataRow dataRow)
++ throws IgniteCheckedException {
++- if (oldRow == null || indexingEnabled)
+++ if (oldRow == null || indexingEnabled || oldRow.expireTime() != dataRow.expireTime())
++ return false;
++
++ CacheObjectContext coCtx = cctx.cacheObjectContext();
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
++index 3bc7004..9297cec 100644
++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
++@@ -130,10 +130,16 @@
++ return cfg;
++ }
++
+++ /**
+++ * @param cfg IgniteConfiguration.
+++ */
++ protected void configure(IgniteConfiguration cfg){
++ //NOP
++ }
++
+++ /**
+++ * @param mCfg MemoryConfiguration.
+++ */
++ protected void configure(MemoryConfiguration mCfg){
++ //NOP
++ }
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
++index fc0e715..bca3af0 100644
++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
++@@ -17,69 +17,157 @@
++
++ package org.apache.ignite.internal.processors.database;
++
++-import org.apache.ignite.Ignite;
++-import org.apache.ignite.IgniteCompute;
++-import org.apache.ignite.compute.ComputeTaskFuture;
++-import org.apache.ignite.internal.IgniteEx;
++-import org.apache.ignite.lang.IgniteRunnable;
++-import org.apache.ignite.resources.IgniteInstanceResource;
++-
+++import java.util.Random;
++ import java.util.concurrent.TimeUnit;
+++import org.apache.ignite.IgniteCache;
+++import org.apache.ignite.configuration.IgniteConfiguration;
+++import org.apache.ignite.configuration.MemoryConfiguration;
+++import org.apache.ignite.internal.IgniteEx;
+++import org.apache.ignite.internal.util.GridRandom;
+++import org.jetbrains.annotations.NotNull;
++
++ /**
++- * TODO: fix javadoc warnings, code style.
+++ * Base class for memory leaks tests.
++ */
++ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest {
++- // TODO: take duration from system property.
++- /** Test duration in seconds*/
++- protected abstract int duration();
++-
++- @Override
++- protected long getTestTimeout() {
++- return duration() * 1200;
++- }
++
++ /** */
++- protected abstract void operation(IgniteEx ig);
+++ private volatile Exception ex = null;
++
++ /** */
++- public void testMemoryLeak() throws Exception {
++- // TODO: take PageMemory max size is the same as we configured.
+++ private static final ThreadLocal<Random> THREAD_LOCAL_RANDOM = new ThreadLocal<>();
++
++- final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration());
+++ /** {@inheritDoc} */
+++ @Override protected void configure(IgniteConfiguration cfg) {
+++ cfg.setMetricsLogFrequency(5000);
+++ }
++
++- // TODO: use threads instead of compute or make sure there are enough threads in pool.
++- int tasksCnt = Runtime.getRuntime().availableProcessors() * 4;
+++ /** {@inheritDoc} */
+++ @Override protected void configure(MemoryConfiguration mCfg) {
+++ int concLvl = Runtime.getRuntime().availableProcessors();
++
++- IgniteCompute compute = grid(0).compute().withAsync();
+++ mCfg.setConcurrencyLevel(concLvl);
+++ mCfg.setPageCacheSize(1024 * 1024 * concLvl); //minimal possible value
+++ }
++
++- ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCnt];
+++ /**
+++ * @return Test duration in seconds.
+++ */
+++ protected int duration() {
+++ return 300;
+++ }
+++
+++ /** {@inheritDoc} */
+++ @Override protected int gridCount() {
+++ return 1;
+++ }
+++
+++ /** {@inheritDoc} */
+++ @Override protected boolean indexingEnabled() {
+++ return false;
+++ }
++
++- for (int i = 0; i < tasksCnt; i++) {
++- compute.run(new IgniteRunnable() {
++- @IgniteInstanceResource
++- private Ignite ig;
+++ /** {@inheritDoc} */
+++ @Override protected long getTestTimeout() {
+++ return (duration() + 1) * 1000;
+++ }
++
++- @Override
++- public void run() {
++- int i = 0;
++- while (System.nanoTime() < end) {
++- operation((IgniteEx) ig);
+++ /**
+++ * @param ig Ignite instance.
+++ * @return IgniteCache.
+++ */
+++ protected abstract IgniteCache<Object,Object> cache(IgniteEx ig);
+++
+++ /**
+++ * @return Cache key to perform an operation.
+++ */
+++ protected abstract Object key();
+++
+++ /**
+++ * @return Cache value to perform an operation.
+++ * @param key Cache key to perform an operation.
+++ */
+++ protected abstract Object value(Object key);
+++
+++ /**
+++ * @param cache IgniteCache.
+++ */
+++ protected void operation(IgniteCache<Object, Object> cache) {
+++ Object key = key();
+++ Object value = value(key);
+++
+++ switch (getRandom().nextInt(3)) {
+++ case 0:
+++ cache.getAndPut(key, value);
+++ case 1:
+++ cache.get(key);
+++ break;
+++ case 2:
+++ cache.getAndRemove(key);
+++ }
+++ }
++
++- if(i++ == 100) {
++- check((IgniteEx) ig);
++- i = 0;
++- }
+++ /**
+++ * @return Random.
+++ */
+++ @NotNull protected static Random getRandom() {
+++ Random rnd = THREAD_LOCAL_RANDOM.get();
+++
+++ if(rnd == null){
+++ rnd = new GridRandom();
+++ THREAD_LOCAL_RANDOM.set(rnd);
+++ }
+++
+++ return rnd;
+++ }
+++
+++ /**
+++ * @throws Exception If failed.
+++ */
+++ public void testMemoryLeak() throws Exception {
+++ final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration());
+++
+++ final IgniteEx ignite = grid(0);
+++ final IgniteCache<Object, Object> cache = cache(ignite);
+++
+++ Runnable target = new Runnable() {
+++ @Override public void run() {
+++ while (ex == null && System.nanoTime() < end) {
+++ try {
+++ operation(cache);
+++ check(ignite);
+++ }
+++ catch (Exception e) {
+++ ex = e;
+++
+++ break;
++ }
++ }
++- });
+++ }
+++ };
+++
+++ Thread[] threads = new Thread[Runtime.getRuntime().availableProcessors()];
++
++- futs[i] = compute.future();
+++ for (int i = 0; i < threads.length; i++) {
+++ threads[i] = new Thread(target);
+++ threads[i].start();
++ }
++
++- for (ComputeTaskFuture fut : futs)
++- fut.get();
+++ for (Thread thread : threads) {
+++ thread.join();
+++ }
+++
+++ if(ex != null){
+++ throw ex;
+++ }
++ }
++
++- protected void check(IgniteEx ig) {}
+++ /**
+++ * Callback to check the current state
+++ *
+++ * @param ig Ignite instance
+++ * @throws Exception If failed.
+++ */
+++ protected void check(IgniteEx ig) throws Exception {
+++ }
++ }
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
++index db77131..acc6c2f 100644
++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
++@@ -17,72 +17,13 @@
++
++ package org.apache.ignite.internal.processors.database;
++
++-import org.apache.ignite.IgniteCache;
++-import org.apache.ignite.configuration.IgniteConfiguration;
++-import org.apache.ignite.configuration.MemoryConfiguration;
++-import org.apache.ignite.internal.IgniteEx;
++-
++-import java.util.Random;
++-import java.util.concurrent.ThreadLocalRandom;
++-
++ /**
++ *
++ */
++-public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakAbstractTest {
++-
++- @Override
++- protected int duration() {
++- return 300;
++- }
++-
++- @Override
++- protected int gridCount() {
++- return 1;
++- }
++-
++- @Override
++- protected void configure(IgniteConfiguration cfg) {
++- cfg.setMetricsLogFrequency(5000);
++- }
++-
++- @Override
++- protected void configure(MemoryConfiguration mCfg) {
++- mCfg.setPageCacheSize(1024 * 1024);
++- }
+++public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakTest {
++
++- @Override
++- // TODO: move test to module ignite-indexing.
++- protected boolean indexingEnabled() {
+++ /** {@inheritDoc} */
+++ @Override protected boolean indexingEnabled() {
++ return true;
++ }
++-
++- protected void operation(IgniteEx ig){
++- IgniteCache<Object, Object> cache = ig.cache("non-primitive");
++- Random rnd = ThreadLocalRandom.current();
++-
++- for (int i = 0; i < 1000; i++) {
++- DbKey key = new DbKey(rnd.nextInt(200_000));
++-
++- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
++-
++- // TODO: also execute sql queries.
++-
++- switch (rnd.nextInt(3)) {
++- case 0:
++- cache.getAndPut(key, v0);
++- case 1:
++- cache.get(key);
++- break;
++- case 2:
++- cache.getAndRemove(key);
++- }
++- }
++- }
++-
++- @Override
++- protected void check(IgniteEx ig) {
++- long pages = ig.context().cache().context().database().pageMemory().loadedPages();
++-
++- assertTrue(pages < 19100);
++- }
++ }
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
++index 2a6c8cd..8943743 100644
++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
++@@ -18,13 +18,8 @@
++ package org.apache.ignite.internal.processors.database;
++
++ import org.apache.ignite.IgniteCache;
++-import org.apache.ignite.configuration.IgniteConfiguration;
++-import org.apache.ignite.configuration.MemoryConfiguration;
++ import org.apache.ignite.internal.IgniteEx;
++
++-import java.util.Random;
++-import java.util.concurrent.ThreadLocalRandom;
++-
++ /**
++ *
++ */
++@@ -35,62 +30,29 @@
++ static {
++ ARRAY = new int[1024];
++
++- Random rnd = new Random();
++-
++ for (int i = 0; i < ARRAY.length; i++)
++- ARRAY[i] = rnd.nextInt();
++- }
++-
++- @Override
++- protected int duration() {
++- return 300;
++- }
++-
++- @Override
++- protected int gridCount() {
++- return 1;
+++ ARRAY[i] = getRandom().nextInt();
++ }
++
++- @Override
++- protected void configure(IgniteConfiguration cfg) {
++- cfg.setMetricsLogFrequency(5000);
+++ /** {@inheritDoc} */
+++ @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
+++ return ig.cache("non-primitive");
++ }
++
++- @Override
++- protected void configure(MemoryConfiguration mCfg) {
++- mCfg.setPageCacheSize(60 * 1024 * 1024);
+++ /** {@inheritDoc} */
+++ @Override protected Object key() {
+++ return new DbKey(getRandom().nextInt(200_000));
++ }
++
++- @Override
++- protected boolean indexingEnabled() {
++- return false;
++- }
++-
++- protected void operation(IgniteEx ig){
++- IgniteCache<Object, Object> cache = ig.cache("large");
++- Random rnd = ThreadLocalRandom.current();
++-
++- for (int i = 0; i < 1000; i++) {
++- LargeDbKey key = new LargeDbKey(rnd.nextInt(10_000), 1024);
++-
++- LargeDbValue v0 = new LargeDbValue("test-value-1-" + rnd.nextInt(200), "test-value-2-" + rnd.nextInt(200), ARRAY);
++-
++- switch (rnd.nextInt(3)) {
++- case 0:
++- cache.getAndPut(key, v0);
++- case 1:
++- cache.get(key);
++- break;
++- case 2:
++- cache.getAndRemove(key);
++- }
++- }
+++ /** {@inheritDoc} */
+++ @Override protected Object value(Object key) {
+++ return new DbValue(((DbKey)key).val, "test-value-" + getRandom().nextInt(200), getRandom().nextInt(500));
++ }
++
++- @Override
++- protected void check(IgniteEx ig) {
+++ /** {@inheritDoc} */
+++ @Override protected void check(IgniteEx ig) {
++ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
++
++- assertTrue(pages < 50000);
+++ assertTrue(pages < 20000);
++ }
++ }
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
++index 91c96af..8e4d0b4 100644
++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
++@@ -17,76 +17,31 @@
++
++ package org.apache.ignite.internal.processors.database;
++
++-import org.apache.ignite.IgniteCache;
++-import org.apache.ignite.configuration.IgniteConfiguration;
++ import org.apache.ignite.configuration.MemoryConfiguration;
++ import org.apache.ignite.internal.IgniteEx;
++
++-import java.util.Random;
++-import java.util.concurrent.ThreadLocalRandom;
++-
++ /**
++ *
++ */
++-public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakAbstractTest {
++-
++- @Override
++- protected int duration() {
++- return 300;
++- }
+++public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakTest {
++
++- @Override
++- protected int gridCount() {
++- return 1;
++- }
+++ /** {@inheritDoc} */
+++ @Override protected void configure(MemoryConfiguration mCfg) {
+++ int concLvl = Runtime.getRuntime().availableProcessors();
+++ mCfg.setConcurrencyLevel(concLvl);
+++ mCfg.setPageCacheSize(1024 * 1024 * concLvl * 16);
++
++- @Override
++- protected void configure(IgniteConfiguration cfg) {
++- cfg.setMetricsLogFrequency(5000);
++ }
++
++- @Override
++- protected void configure(MemoryConfiguration mCfg) {
++- // TODO: understand why such overhead with large pages.
++- mCfg.setPageCacheSize(100 * 1024 * 1024);
++- }
++-
++- @Override
++- protected boolean indexingEnabled() {
++- return false;
++- }
++-
++- @Override
++- protected boolean isLargePage() {
+++ /** {@inheritDoc} */
+++ @Override protected boolean isLargePage() {
++ return true;
++ }
++
++- // TODO: avoid copy/paste.
++- protected void operation(IgniteEx ig){
++- IgniteCache<Object, Object> cache = ig.cache("non-primitive");
++- Random rnd = ThreadLocalRandom.current();
++-
++- for (int i = 0; i < 1000; i++) {
++- DbKey key = new DbKey(rnd.nextInt(200_000));
++-
++- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
++-
++- switch (rnd.nextInt(3)) {
++- case 0:
++- cache.getAndPut(key, v0);
++- case 1:
++- cache.get(key);
++- break;
++- case 2:
++- cache.getAndRemove(key);
++- }
++- }
++- }
++-
++- @Override
++- protected void check(IgniteEx ig) {
+++ /** {@inheritDoc} */
+++ @Override protected void check(IgniteEx ig) {
++ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
++
++- assertTrue(pages < 4600);
+++ assertTrue(pages < 4000);
++ }
++ }
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
++index 2b0ce1e..81d831b 100644
++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
++@@ -18,72 +18,31 @@
++ package org.apache.ignite.internal.processors.database;
++
++ import org.apache.ignite.IgniteCache;
++-import org.apache.ignite.configuration.IgniteConfiguration;
++-import org.apache.ignite.configuration.MemoryConfiguration;
++ import org.apache.ignite.internal.IgniteEx;
++
++-import java.util.Random;
++-import java.util.concurrent.ThreadLocalRandom;
++-
++ /**
++ *
++ */
++ public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
++ /** {@inheritDoc} */
++- @Override protected int duration() {
++- return 300;
++- }
++-
++- @Override
++- protected int gridCount() {
++- return 1;
++- }
++-
++- @Override
++- protected void configure(IgniteConfiguration cfg) {
++- cfg.setMetricsLogFrequency(5000);
++- }
++-
++- @Override
++- protected void configure(MemoryConfiguration mCfg) {
++- mCfg.setPageCacheSize(1024 * 1024);
+++ @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
+++ return ig.cache("non-primitive");
++ }
++
++- @Override
++- protected boolean indexingEnabled() {
++- return false;
+++ /** {@inheritDoc} */
+++ @Override protected Object key() {
+++ return new DbKey(getRandom().nextInt(200_000));
++ }
++
++- protected void operation(IgniteEx ig){
++- IgniteCache<Object, Object> cache = ig.cache("non-primitive");
++- Random rnd = ThreadLocalRandom.current();
++-
++- for (int i = 0; i < 1000; i++) {
++- DbKey key = new DbKey(rnd.nextInt(200_000));
++-
++- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
++-
++- // TODO: also execute scan query.
++-
++- switch (rnd.nextInt(3)) {
++- case 0:
++- cache.getAndPut(key, v0);
++- break;
++-
++- case 1:
++- cache.get(key);
++- break;
++-
++- case 2:
++- cache.getAndRemove(key);
++- }
++- }
+++ /** {@inheritDoc} */
+++ @Override protected Object value(Object key) {
+++ return new DbValue(((DbKey)key).val, "test-value-" + getRandom().nextInt(200), getRandom().nextInt(500));
++ }
++
++- @Override
++- protected void check(IgniteEx ig) {
+++ /** {@inheritDoc} */
+++ @Override protected void check(IgniteEx ig) {
++ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
++
++- assertTrue(pages < 19100);
+++ assertTrue(pages < 20000);
++ }
++ }
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
++index 95fe8c8..a31ffb4 100644
++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
++@@ -18,75 +18,31 @@
++ package org.apache.ignite.internal.processors.database;
++
++ import org.apache.ignite.IgniteCache;
++-import org.apache.ignite.configuration.IgniteConfiguration;
++ import org.apache.ignite.configuration.MemoryConfiguration;
++ import org.apache.ignite.internal.IgniteEx;
++
++ import javax.cache.expiry.CreatedExpiryPolicy;
++ import javax.cache.expiry.Duration;
++ import javax.cache.expiry.ExpiryPolicy;
++-import java.util.Random;
++-import java.util.concurrent.ThreadLocalRandom;
++
++ import static java.util.concurrent.TimeUnit.MILLISECONDS;
++
++ /**
++ *
++ */
++-public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakAbstractTest {
+++public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakTest {
++ /** */
++ private static final ExpiryPolicy EXPIRY = new CreatedExpiryPolicy(new Duration(MILLISECONDS, 10L));
++
++- @Override
++- protected int duration() {
++- return 300;
+++ /** {@inheritDoc} */
+++ @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
+++ return ig.cache("non-primitive").withExpiryPolicy(EXPIRY);
++ }
++
++- @Override
++- protected int gridCount() {
++- return 1;
++- }
++-
++- @Override
++- protected void configure(IgniteConfiguration cfg) {
++- cfg.setMetricsLogFrequency(5000);
++- }
++-
++- @Override
++- protected void configure(MemoryConfiguration mCfg) {
++- mCfg.setPageCacheSize(1024 * 1024);
++- }
++-
++- @Override
++- protected boolean indexingEnabled() {
++- return false;
++- }
++-
++- protected void operation(IgniteEx ig) {
++- IgniteCache<Object, Object> cache = ig.cache("non-primitive").withExpiryPolicy(EXPIRY);
++- Random rnd = ThreadLocalRandom.current();
++-
++- for (int i = 0; i < 1000; i++) {
++- DbKey key = new DbKey(rnd.nextInt(200_000));
++-
++- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
++-
++- switch (rnd.nextInt(3)) {
++- case 0:
++- cache.getAndPut(key, v0);
++- case 1:
++- cache.get(key);
++- break;
++- case 2:
++- cache.getAndRemove(key);
++- }
++- }
++- }
++-
++- @Override
++- protected void check(IgniteEx ig) {
+++ /** {@inheritDoc} */
+++ @Override protected void check(IgniteEx ig) {
++ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
++
++- assertTrue(pages < 10000);
+++ assertTrue(pages < 7000);
++ }
++ }
++
++From 84c03e0c522abc90b2d91e514138eac08388abd2 Mon Sep 17 00:00:00 2001
++From: Igor Seliverstov <gv...@gmail.com>
++Date: Thu, 16 Feb 2017 13:41:51 +0300
++Subject: [PATCH 04/41] IGNITE-4694 Add tests to check there are no memory
++ leaks in PageMemory (pending)
++
++---
++ .../database/IgniteDbMemoryLeakAbstractTest.java | 118 +++++++++++++++++----
++ .../database/IgniteDbMemoryLeakIndexedTest.java | 29 -----
++ .../IgniteDbMemoryLeakLargeObjectsTest.java | 16 +--
++ .../database/IgniteDbMemoryLeakLargePagesTest.java | 13 +--
++ .../database/IgniteDbMemoryLeakTest.java | 14 ++-
++ .../IgniteDbMemoryLeakWithExpirationTest.java | 7 +-
++ .../database/IgniteDbMemoryLeakIndexedTest.java | 42 ++++++++
++ 7 files changed, 165 insertions(+), 74 deletions(-)
++ delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
++ create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
++
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
++index bca3af0..819405e 100644
++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
++@@ -30,13 +30,37 @@
++ * Base class for memory leaks tests.
++ */
++ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest {
++-
+++ /** */
+++ @SuppressWarnings("WeakerAccess") protected static final int CONCURRENCY_LEVEL = 8;
++ /** */
++ private volatile Exception ex = null;
++
++ /** */
+++ private long warmUpEndTime;
+++
+++ /** */
+++ private long endTime;
+++
+++ /** */
+++ private long loadedPages = 0;
+++
+++ /** */
+++ private long delta = 0;
+++
+++ /** */
+++ private long probeCnt = 0;
+++
+++ /** */
++ private static final ThreadLocal<Random> THREAD_LOCAL_RANDOM = new ThreadLocal<>();
++
+++ @Override protected void beforeTest() throws Exception {
+++ super.beforeTest();
+++
+++ long startTime = System.nanoTime();
+++ warmUpEndTime = startTime + TimeUnit.SECONDS.toNanos(warmUp());
+++ endTime = warmUpEndTime + TimeUnit.SECONDS.toNanos(duration());
+++ }
+++
++ /** {@inheritDoc} */
++ @Override protected void configure(IgniteConfiguration cfg) {
++ cfg.setMetricsLogFrequency(5000);
++@@ -44,10 +68,7 @@
++
++ /** {@inheritDoc} */
++ @Override protected void configure(MemoryConfiguration mCfg) {
++- int concLvl = Runtime.getRuntime().availableProcessors();
++-
++- mCfg.setConcurrencyLevel(concLvl);
++- mCfg.setPageCacheSize(1024 * 1024 * concLvl); //minimal possible value
+++ mCfg.setConcurrencyLevel(CONCURRENCY_LEVEL);
++ }
++
++ /**
++@@ -57,6 +78,13 @@ protected int duration() {
++ return 300;
++ }
++
+++ /**
+++ * @return Warm up duration.
+++ */
+++ @SuppressWarnings("WeakerAccess") protected int warmUp() {
+++ return 300;
+++ }
+++
++ /** {@inheritDoc} */
++ @Override protected int gridCount() {
++ return 1;
++@@ -69,14 +97,14 @@ protected int duration() {
++
++ /** {@inheritDoc} */
++ @Override protected long getTestTimeout() {
++- return (duration() + 1) * 1000;
+++ return (warmUp() + duration() + 1) * 1000; // One extra second to stop all threads
++ }
++
++ /**
++ * @param ig Ignite instance.
++ * @return IgniteCache.
++ */
++- protected abstract IgniteCache<Object,Object> cache(IgniteEx ig);
+++ protected abstract IgniteCache<Object, Object> cache(IgniteEx ig);
++
++ /**
++ * @return Cache key to perform an operation.
++@@ -84,8 +112,8 @@ protected int duration() {
++ protected abstract Object key();
++
++ /**
++- * @return Cache value to perform an operation.
++ * @param key Cache key to perform an operation.
+++ * @return Cache value to perform an operation.
++ */
++ protected abstract Object value(Object key);
++
++@@ -99,8 +127,11 @@ protected void operation(IgniteCache<Object, Object> cache) {
++ switch (getRandom().nextInt(3)) {
++ case 0:
++ cache.getAndPut(key, value);
+++
+++ break;
++ case 1:
++ cache.get(key);
+++
++ break;
++ case 2:
++ cache.getAndRemove(key);
++@@ -113,7 +144,7 @@ protected void operation(IgniteCache<Object, Object> cache) {
++ @NotNull protected static Random getRandom() {
++ Random rnd = THREAD_LOCAL_RANDOM.get();
++
++- if(rnd == null){
+++ if (rnd == null) {
++ rnd = new GridRandom();
++ THREAD_LOCAL_RANDOM.set(rnd);
++ }
++@@ -125,49 +156,96 @@ protected void operation(IgniteCache<Object, Object> cache) {
++ * @throws Exception If failed.
++ */
++ public void testMemoryLeak() throws Exception {
++- final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration());
++
++ final IgniteEx ignite = grid(0);
++ final IgniteCache<Object, Object> cache = cache(ignite);
++
++ Runnable target = new Runnable() {
++ @Override public void run() {
++- while (ex == null && System.nanoTime() < end) {
+++ while (ex == null && System.nanoTime() < endTime) {
++ try {
++ operation(cache);
++- check(ignite);
++ }
++ catch (Exception e) {
++ ex = e;
++-
++ break;
++ }
++ }
++ }
++ };
++
++- Thread[] threads = new Thread[Runtime.getRuntime().availableProcessors()];
+++ Thread[] threads = new Thread[CONCURRENCY_LEVEL];
+++
+++ info("Warming up is started.");
++
++ for (int i = 0; i < threads.length; i++) {
++ threads[i] = new Thread(target);
++ threads[i].start();
++ }
++
++- for (Thread thread : threads) {
++- thread.join();
+++ Thread.sleep(TimeUnit.NANOSECONDS.toMillis(warmUpEndTime - System.nanoTime()));
+++
+++ info("Warming up is ended.");
+++
+++ while (System.nanoTime() < endTime) {
+++ try {
+++ check(ignite);
+++ }
+++ catch (Exception e) {
+++ ex = e;
+++
+++ break;
+++ }
+++
+++ Thread.sleep(TimeUnit.SECONDS.toMillis(5));
++ }
++
++- if(ex != null){
+++ for (Thread thread : threads)
+++ thread.join();
+++
+++ if (ex != null)
++ throw ex;
++- }
++ }
++
++ /**
++- * Callback to check the current state
+++ * Callback to check the current state.
++ *
++- * @param ig Ignite instance
+++ * @param ig Ignite instance.
++ * @throws Exception If failed.
++ */
++ protected void check(IgniteEx ig) throws Exception {
+++ long pagesActual = ig.context().cache().context().database().pageMemory().loadedPages();
+++ long pagesMax = pagesMax();
+++
+++ assertTrue(
+++ "Maximal allowed pages number is exceeded. [allowed=" + pagesMax + "; actual= " + pagesActual + "]",
+++ pagesActual <= pagesMax);
+++
+++ if (loadedPages > 0) {
+++ delta += pagesActual - loadedPages;
+++ int allowedDelta = pagesDelta();
+++
+++ if(probeCnt++ > 12) { // we need some statistic first. Minimal statistic is taken for a minute.
+++ long actualDelta = delta / probeCnt;
+++
+++ assertTrue(
+++ "Average growth pages in the number is more than expected. [allowed=" + allowedDelta + "; actual=" + actualDelta + "]",
+++ actualDelta <= allowedDelta);
+++ }
+++ }
+++
+++ loadedPages = pagesActual;
+++ }
+++
+++ /**
+++ * @return Maximal allowed pages number.
+++ */
+++ protected abstract long pagesMax();
+++
+++ /**
+++ * @return Expected average number of pages, on which their total number can grow per 5 seconds.
+++ */
+++ @SuppressWarnings("WeakerAccess") protected int pagesDelta() {
+++ return 5;
++ }
++ }
++diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
++deleted file mode 100644
++index acc6c2f..0000000
++--- a/modules/core/src/test/java/org/apache/ignite/internal/processors
<TRUNCATED>
http://git-wip-us.apache.org/repos/asf/ignite/blob/81ae2d83/modules/clients/src/test/java/org/apache/ignite/internal/client/integration/ClientAbstractSelfTest.java
----------------------------------------------------------------------