You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by se...@apache.org on 2020/07/24 14:55:05 UTC
[ignite] branch master updated: IGNITE-13263 PDS4 suite
stabilization - timeouts and memory usage improvements - Fixes #8045.
This is an automated email from the ASF dual-hosted git repository.
sergeychugunov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new f0ad15c IGNITE-13263 PDS4 suite stabilization - timeouts and memory usage improvements - Fixes #8045.
f0ad15c is described below
commit f0ad15c36044b40e53cb45ddc27c27dbc16821e2
Author: Nikita Tolstunov <nt...@gmail.com>
AuthorDate: Fri Jul 24 17:53:17 2020 +0300
IGNITE-13263 PDS4 suite stabilization - timeouts and memory usage improvements - Fixes #8045.
Signed-off-by: Sergey Chugunov <se...@gmail.com>
---
.../pagelocktracker/AbstractPageLockTest.java | 11 +++-
.../PageLockTrackerManagerTest.java | 9 ++-
.../pagelocktracker/SharedPageLockTrackerTest.java | 70 ++++++++++++++--------
.../dumpprocessors/ToFileDumpProcessorTest.java | 16 ++---
.../testframework/junits/GridAbstractTest.java | 53 ++++++++++++++++
5 files changed, 119 insertions(+), 40 deletions(-)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/diagnostic/pagelocktracker/AbstractPageLockTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/diagnostic/pagelocktracker/AbstractPageLockTest.java
index 4ac9c82..bd1f372 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/diagnostic/pagelocktracker/AbstractPageLockTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/diagnostic/pagelocktracker/AbstractPageLockTest.java
@@ -22,11 +22,16 @@ import org.apache.ignite.IgniteException;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.log.PageLockLogSnapshot;
import org.apache.ignite.internal.util.typedef.internal.U;
-
-import static org.junit.Assert.assertEquals;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
/** */
-public abstract class AbstractPageLockTest {
+public abstract class AbstractPageLockTest extends GridCommonAbstractTest {
+
+ /** */
+ protected AbstractPageLockTest() {
+ super(false);
+ }
+
/** */
protected void doRunnable(int deep, Runnable r) {
for (int i = 0; i < deep; i++)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/diagnostic/pagelocktracker/PageLockTrackerManagerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/diagnostic/pagelocktracker/PageLockTrackerManagerTest.java
index 1d95dff..ea71e91 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/diagnostic/pagelocktracker/PageLockTrackerManagerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/diagnostic/pagelocktracker/PageLockTrackerManagerTest.java
@@ -24,6 +24,7 @@ import org.apache.ignite.internal.processors.cache.persistence.DataStructure;
import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageLockListener;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.testframework.ListeningTestLogger;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.junit.Assert;
import org.junit.Test;
@@ -35,7 +36,13 @@ import static org.apache.ignite.internal.processors.cache.persistence.diagnostic
/**
*
*/
-public class PageLockTrackerManagerTest {
+public class PageLockTrackerManagerTest extends GridCommonAbstractTest {
+
+ /** */
+ public PageLockTrackerManagerTest() {
+ super(false);
+ }
+
/**
*
*/
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/diagnostic/pagelocktracker/SharedPageLockTrackerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/diagnostic/pagelocktracker/SharedPageLockTrackerTest.java
index fda21a6..c70018f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/diagnostic/pagelocktracker/SharedPageLockTrackerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/diagnostic/pagelocktracker/SharedPageLockTrackerTest.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -36,42 +37,63 @@ import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.GridTestUtils.SF;
import org.junit.Assert;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import static org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.LockTrackerFactory.HEAP_LOG;
import static org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.LockTrackerFactory.HEAP_STACK;
import static org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.LockTrackerFactory.OFF_HEAP_LOG;
import static org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.LockTrackerFactory.OFF_HEAP_STACK;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
/**
*
*/
+@RunWith(Parameterized.class)
public class SharedPageLockTrackerTest extends AbstractPageLockTest {
+
+ /** Tracker types. */
+ @Parameterized.Parameter(0)
+ public int trackerType;
+
+ /** */
+ private final int defaultType = LockTrackerFactory.DEFAULT_TYPE;
+
/**
- *
+ * @return Test parameters.
*/
+ @Parameterized.Parameters(name = "trackerType={0}")
+ public static Collection<Object[]> getParameters() {
+ Collection<Object[]> params = new ArrayList<>();
+
+ params.add(new Object[]{HEAP_STACK});
+ params.add(new Object[]{HEAP_LOG});
+ params.add(new Object[]{OFF_HEAP_STACK});
+ params.add(new Object[]{OFF_HEAP_LOG});
+
+ return params;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+ LockTrackerFactory.DEFAULT_TYPE = defaultType;
+ }
+
+ /** */
@Test
public void testTakeDumpByCount() throws Exception {
- int[] trackerTypes = new int[] {HEAP_STACK, HEAP_LOG, OFF_HEAP_STACK, OFF_HEAP_LOG};
-
LockTrackerFactory.DEFAULT_CAPACITY = 512;
- for (int i = 0; i < trackerTypes.length; i++) {
- LockTrackerFactory.DEFAULT_TYPE = trackerTypes[i];
-
- int dumps = SF.apply(30, 10, 40);
+ LockTrackerFactory.DEFAULT_TYPE = trackerType;
- doTestTakeDumpByCount(5, 1, dumps, 1);
+ int dumps = SF.apply(5, 2, 10);
+ doTestTakeDumpByCount(5, 1, dumps, 1);
- doTestTakeDumpByCount(5, 2, dumps, 2);
+ doTestTakeDumpByCount(5, 2, dumps, 2);
- doTestTakeDumpByCount(10, 3, dumps, 4);
+ doTestTakeDumpByCount(10, 3, dumps, 4);
- doTestTakeDumpByCount(20, 6, dumps, 8);
- }
+ doTestTakeDumpByCount(20, 6, dumps, 8);
}
/**
@@ -79,23 +101,19 @@ public class SharedPageLockTrackerTest extends AbstractPageLockTest {
*/
@Test
public void testTakeDumpByTime() throws Exception {
- int[] trackerTypes = new int[] {HEAP_STACK, HEAP_LOG, OFF_HEAP_STACK, OFF_HEAP_LOG};
-
LockTrackerFactory.DEFAULT_CAPACITY = 512;
- for (int i = 0; i < trackerTypes.length; i++) {
- LockTrackerFactory.DEFAULT_TYPE = trackerTypes[i];
+ LockTrackerFactory.DEFAULT_TYPE = trackerType;
- int time = SF.apply(30_000, 5_000, 40_000);
+ int time = SF.apply(3_000, 1_000, 5_000);
- doTestTakeDumpByTime(5, 1, time, 1);
+ doTestTakeDumpByTime(5, 1, time, 1);
- doTestTakeDumpByTime(5, 2, time, 2);
+ doTestTakeDumpByTime(5, 2, time, 2);
- doTestTakeDumpByTime(10, 3, time, 4);
+ doTestTakeDumpByTime(10, 3, time, 4);
- doTestTakeDumpByTime(20, 6, time, 8);
- }
+ doTestTakeDumpByTime(20, 6, time, 8);
}
/**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/diagnostic/pagelocktracker/dumpprocessors/ToFileDumpProcessorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/diagnostic/pagelocktracker/dumpprocessors/ToFileDumpProcessorTest.java
index 1c7c65d..ae05e44 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/diagnostic/pagelocktracker/dumpprocessors/ToFileDumpProcessorTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/diagnostic/pagelocktracker/dumpprocessors/ToFileDumpProcessorTest.java
@@ -25,9 +25,8 @@ import org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelo
import org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockDump;
import org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTracker;
import org.apache.ignite.internal.util.typedef.internal.U;
-import org.junit.After;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.junit.Assert;
-import org.junit.Before;
import org.junit.Test;
import static java.nio.file.Paths.get;
@@ -35,18 +34,15 @@ import static java.nio.file.Paths.get;
/**
*
*/
-public class ToFileDumpProcessorTest {
+public class ToFileDumpProcessorTest extends GridCommonAbstractTest {
/** */
private File file;
- @Before
- public void beforeTest() {
- cleanFile();
- }
+ /** {@inheritDoc} */
+ @Override public void beforeTest() throws Exception {
+ super.beforeTest();
- @After
- public void afterTest() {
- // cleanFile();
+ cleanFile();
}
/**
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index b237be7..1b8d866 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -21,6 +21,8 @@ import java.io.ObjectStreamException;
import java.io.Serializable;
import java.lang.annotation.Annotation;
import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.management.MemoryUsage;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationHandler;
@@ -199,6 +201,12 @@ public abstract class GridAbstractTest extends JUnitAssertAware {
private static final transient Map<Logger, Level> changedLevels = new ConcurrentHashMap<>();
/** */
+ private static final MemoryMXBean memoryMxBean = ManagementFactory.getMemoryMXBean();
+
+ /** */
+ private static final int BYTES_IN_MEGABYTE = 1024 * 1024;
+
+ /** */
protected static final String DEFAULT_CACHE_NAME = "default";
/** Sustains {@link #beforeTestsStarted()} and {@link #afterTestsStopped()} methods execution.*/
@@ -721,6 +729,7 @@ public abstract class GridAbstractTest extends JUnitAssertAware {
long dur = System.currentTimeMillis() - ts;
U.quietAndInfo(log(),">>> Stopping test: " + testDescription() + " in " + dur + " ms <<<");
+ printJvmMemoryStatistic();
try {
afterTest();
@@ -737,6 +746,22 @@ public abstract class GridAbstractTest extends JUnitAssertAware {
}
}
+ /** Prints JVM memory statistic. */
+ private void printJvmMemoryStatistic() {
+ U.quietAndInfo(log(),
+ ">>> Heap: " + createPrettyMemoryString(memoryMxBean.getHeapMemoryUsage()) + " <<<");
+
+ U.quietAndInfo(log(),
+ ">>> Non-Heap: " + createPrettyMemoryString(memoryMxBean.getNonHeapMemoryUsage()) + " <<<");
+ }
+
+ /** */
+ private static String createPrettyMemoryString(MemoryUsage usage) {
+ return usage.getUsed() / BYTES_IN_MEGABYTE + " MB used / "
+ + usage.getCommitted() / BYTES_IN_MEGABYTE + " MB commited / "
+ + usage.getMax() / BYTES_IN_MEGABYTE + " MB max";
+ }
+
/**
* @return Test description.
*/
@@ -1135,6 +1160,8 @@ public abstract class GridAbstractTest extends JUnitAssertAware {
*/
protected Ignite startGrid(String igniteInstanceName, IgniteConfiguration cfg, GridSpringResourceContext ctx)
throws Exception {
+ limitMaxMemoryOfDataStorageConfiguration(cfg);
+
if (!isRemoteJvm(igniteInstanceName)) {
IgniteUtils.setCurrentIgniteName(igniteInstanceName);
@@ -1180,6 +1207,31 @@ public abstract class GridAbstractTest extends JUnitAssertAware {
}
/**
+ * Limits max memory in data storage if they are default.
+ * By default max memory is 20% of total memory, which is too much for tests.
+ * Required for containerized build servers.
+ */
+ private void limitMaxMemoryOfDataStorageConfiguration(IgniteConfiguration cfg) {
+ DataStorageConfiguration dataStorageCfg = cfg.getDataStorageConfiguration();
+
+ if (dataStorageCfg == null)
+ return;
+
+ updateMaxSizeInRegion(dataStorageCfg.getDefaultDataRegionConfiguration());
+
+ if (dataStorageCfg.getDataRegionConfigurations() != null) {
+ for (DataRegionConfiguration configuration : dataStorageCfg.getDataRegionConfigurations())
+ updateMaxSizeInRegion(configuration);
+ }
+ }
+
+ /** */
+ private static void updateMaxSizeInRegion(DataRegionConfiguration configuration) {
+ if (configuration.getMaxSize() == DataStorageConfiguration.DFLT_DATA_REGION_MAX_SIZE)
+ configuration.setMaxSize(256 * BYTES_IN_MEGABYTE);
+ }
+
+ /**
* Starts new grid at another JVM with given name.
*
* @param igniteInstanceName Ignite instance name.
@@ -2351,6 +2403,7 @@ public abstract class GridAbstractTest extends JUnitAssertAware {
LT.clear();
U.quietAndInfo(log(), ">>> Starting test: " + testDescription() + " <<<");
+ printJvmMemoryStatistic();
try {
beforeTest();