You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2017/03/10 07:23:45 UTC
[24/43] ignite git commit: IGNITE-4694 Add tests to check there are
no memory leaks in PageMemory (pending)
IGNITE-4694 Add tests to check there are no memory leaks in PageMemory (pending)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6743d5c6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6743d5c6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6743d5c6
Branch: refs/heads/ignite-4712
Commit: 6743d5c6b7f7a8b71b9560acf06a86d6cb031523
Parents: 25c1927
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Thu Feb 16 13:41:51 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:54 2017 +0300
----------------------------------------------------------------------
.../IgniteDbMemoryLeakAbstractTest.java | 118 +++++++++++++++----
.../database/IgniteDbMemoryLeakIndexedTest.java | 29 -----
.../IgniteDbMemoryLeakLargeObjectsTest.java | 16 +--
.../IgniteDbMemoryLeakLargePagesTest.java | 13 +-
.../database/IgniteDbMemoryLeakTest.java | 14 ++-
.../IgniteDbMemoryLeakWithExpirationTest.java | 7 +-
.../database/IgniteDbMemoryLeakIndexedTest.java | 42 +++++++
7 files changed, 165 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index bca3af0..819405e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -30,13 +30,37 @@ import org.jetbrains.annotations.NotNull;
* Base class for memory leaks tests.
*/
public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest {
-
+ /** */
+ @SuppressWarnings("WeakerAccess") protected static final int CONCURRENCY_LEVEL = 8;
/** */
private volatile Exception ex = null;
/** */
+ private long warmUpEndTime;
+
+ /** */
+ private long endTime;
+
+ /** */
+ private long loadedPages = 0;
+
+ /** */
+ private long delta = 0;
+
+ /** */
+ private long probeCnt = 0;
+
+ /** */
private static final ThreadLocal<Random> THREAD_LOCAL_RANDOM = new ThreadLocal<>();
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ long startTime = System.nanoTime();
+ warmUpEndTime = startTime + TimeUnit.SECONDS.toNanos(warmUp());
+ endTime = warmUpEndTime + TimeUnit.SECONDS.toNanos(duration());
+ }
+
/** {@inheritDoc} */
@Override protected void configure(IgniteConfiguration cfg) {
cfg.setMetricsLogFrequency(5000);
@@ -44,10 +68,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/** {@inheritDoc} */
@Override protected void configure(MemoryConfiguration mCfg) {
- int concLvl = Runtime.getRuntime().availableProcessors();
-
- mCfg.setConcurrencyLevel(concLvl);
- mCfg.setPageCacheSize(1024 * 1024 * concLvl); //minimal possible value
+ mCfg.setConcurrencyLevel(CONCURRENCY_LEVEL);
}
/**
@@ -57,6 +78,13 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
return 300;
}
+ /**
+ * @return Warm up duration.
+ */
+ @SuppressWarnings("WeakerAccess") protected int warmUp() {
+ return 300;
+ }
+
/** {@inheritDoc} */
@Override protected int gridCount() {
return 1;
@@ -69,14 +97,14 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/** {@inheritDoc} */
@Override protected long getTestTimeout() {
- return (duration() + 1) * 1000;
+ return (warmUp() + duration() + 1) * 1000; // One extra second to stop all threads
}
/**
* @param ig Ignite instance.
* @return IgniteCache.
*/
- protected abstract IgniteCache<Object,Object> cache(IgniteEx ig);
+ protected abstract IgniteCache<Object, Object> cache(IgniteEx ig);
/**
* @return Cache key to perform an operation.
@@ -84,8 +112,8 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
protected abstract Object key();
/**
- * @return Cache value to perform an operation.
* @param key Cache key to perform an operation.
+ * @return Cache value to perform an operation.
*/
protected abstract Object value(Object key);
@@ -99,8 +127,11 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
switch (getRandom().nextInt(3)) {
case 0:
cache.getAndPut(key, value);
+
+ break;
case 1:
cache.get(key);
+
break;
case 2:
cache.getAndRemove(key);
@@ -113,7 +144,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
@NotNull protected static Random getRandom() {
Random rnd = THREAD_LOCAL_RANDOM.get();
- if(rnd == null){
+ if (rnd == null) {
rnd = new GridRandom();
THREAD_LOCAL_RANDOM.set(rnd);
}
@@ -125,49 +156,96 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
* @throws Exception If failed.
*/
public void testMemoryLeak() throws Exception {
- final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration());
final IgniteEx ignite = grid(0);
final IgniteCache<Object, Object> cache = cache(ignite);
Runnable target = new Runnable() {
@Override public void run() {
- while (ex == null && System.nanoTime() < end) {
+ while (ex == null && System.nanoTime() < endTime) {
try {
operation(cache);
- check(ignite);
}
catch (Exception e) {
ex = e;
-
break;
}
}
}
};
- Thread[] threads = new Thread[Runtime.getRuntime().availableProcessors()];
+ Thread[] threads = new Thread[CONCURRENCY_LEVEL];
+
+ info("Warming up is started.");
for (int i = 0; i < threads.length; i++) {
threads[i] = new Thread(target);
threads[i].start();
}
- for (Thread thread : threads) {
- thread.join();
+ Thread.sleep(TimeUnit.NANOSECONDS.toMillis(warmUpEndTime - System.nanoTime()));
+
+ info("Warming up is ended.");
+
+ while (System.nanoTime() < endTime) {
+ try {
+ check(ignite);
+ }
+ catch (Exception e) {
+ ex = e;
+
+ break;
+ }
+
+ Thread.sleep(TimeUnit.SECONDS.toMillis(5));
}
- if(ex != null){
+ for (Thread thread : threads)
+ thread.join();
+
+ if (ex != null)
throw ex;
- }
}
/**
- * Callback to check the current state
+ * Callback to check the current state.
*
- * @param ig Ignite instance
+ * @param ig Ignite instance.
* @throws Exception If failed.
*/
protected void check(IgniteEx ig) throws Exception {
+ long pagesActual = ig.context().cache().context().database().pageMemory().loadedPages();
+ long pagesMax = pagesMax();
+
+ assertTrue(
+ "Maximal allowed pages number is exceeded. [allowed=" + pagesMax + "; actual= " + pagesActual + "]",
+ pagesActual <= pagesMax);
+
+ if (loadedPages > 0) {
+ delta += pagesActual - loadedPages;
+ int allowedDelta = pagesDelta();
+
+ if(probeCnt++ > 12) { // we need some statistic first. Minimal statistic is taken for a minute.
+ long actualDelta = delta / probeCnt;
+
+ assertTrue(
+ "Average growth pages in the number is more than expected. [allowed=" + allowedDelta + "; actual=" + actualDelta + "]",
+ actualDelta <= allowedDelta);
+ }
+ }
+
+ loadedPages = pagesActual;
+ }
+
+ /**
+ * @return Maximal allowed pages number.
+ */
+ protected abstract long pagesMax();
+
+ /**
+ * @return Expected average number of pages, on which their total number can grow per 5 seconds.
+ */
+ @SuppressWarnings("WeakerAccess") protected int pagesDelta() {
+ return 5;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
deleted file mode 100644
index acc6c2f..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.database;
-
-/**
- *
- */
-public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakTest {
-
- /** {@inheritDoc} */
- @Override protected boolean indexingEnabled() {
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
index 8943743..c1b1c9b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
@@ -36,23 +36,25 @@ public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstra
/** {@inheritDoc} */
@Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
- return ig.cache("non-primitive");
+ return ig.cache("large");
}
/** {@inheritDoc} */
@Override protected Object key() {
- return new DbKey(getRandom().nextInt(200_000));
+ return new LargeDbKey(getRandom().nextInt(10_000), 1024);
}
/** {@inheritDoc} */
@Override protected Object value(Object key) {
- return new DbValue(((DbKey)key).val, "test-value-" + getRandom().nextInt(200), getRandom().nextInt(500));
+ return new LargeDbValue("test-value-1-" + getRandom().nextInt(200), "test-value-2-" + getRandom().nextInt(200), ARRAY);
}
- /** {@inheritDoc} */
- @Override protected void check(IgniteEx ig) {
- long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+ @Override protected int warmUp() {
+ return 600;
+ }
- assertTrue(pages < 20000);
+ /** {@inheritDoc} */
+ @Override protected long pagesMax() {
+ return 2000000;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
index 8e4d0b4..9dd9fdc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
@@ -18,19 +18,16 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.internal.IgniteEx;
/**
*
*/
public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakTest {
-
/** {@inheritDoc} */
@Override protected void configure(MemoryConfiguration mCfg) {
- int concLvl = Runtime.getRuntime().availableProcessors();
- mCfg.setConcurrencyLevel(concLvl);
- mCfg.setPageCacheSize(1024 * 1024 * concLvl * 16);
+ super.configure(mCfg);
+ mCfg.setPageCacheSize(67108864); // The space for 4000 large pages
}
/** {@inheritDoc} */
@@ -39,9 +36,7 @@ public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakTest {
}
/** {@inheritDoc} */
- @Override protected void check(IgniteEx ig) {
- long pages = ig.context().cache().context().database().pageMemory().loadedPages();
-
- assertTrue(pages < 4000);
+ @Override protected long pagesMax() {
+ return 4000;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
index 81d831b..94e419d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
/**
@@ -25,6 +26,13 @@ import org.apache.ignite.internal.IgniteEx;
*/
public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
/** {@inheritDoc} */
+ @Override protected void configure(MemoryConfiguration mCfg) {
+ super.configure(mCfg);
+
+ mCfg.setPageCacheSize(20971520); // The space for 20000 pages
+ }
+
+ /** {@inheritDoc} */
@Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
return ig.cache("non-primitive");
}
@@ -40,9 +48,7 @@ public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
}
/** {@inheritDoc} */
- @Override protected void check(IgniteEx ig) {
- long pages = ig.context().cache().context().database().pageMemory().loadedPages();
-
- assertTrue(pages < 20000);
+ @Override protected long pagesMax() {
+ return 20_000;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
index a31ffb4..c1e279b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
@@ -39,10 +39,7 @@ public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakTest
return ig.cache("non-primitive").withExpiryPolicy(EXPIRY);
}
- /** {@inheritDoc} */
- @Override protected void check(IgniteEx ig) {
- long pages = ig.context().cache().context().database().pageMemory().loadedPages();
-
- assertTrue(pages < 7000);
+ @Override protected long pagesMax() {
+ return 7000;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
new file mode 100644
index 0000000..c917bc1
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.configuration.MemoryConfiguration;
+
+/**
+ *
+ */
+public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakTest {
+ /** {@inheritDoc} */
+ @Override protected void configure(MemoryConfiguration mCfg) {
+ super.configure(mCfg);
+
+ mCfg.setPageCacheSize(25165824); // The space for 24000 pages
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean indexingEnabled() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long pagesMax() {
+ return 24_000;
+ }
+}