You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2017/03/10 07:23:22 UTC
[01/43] ignite git commit: IGNITE-4694 Add tests to check there are
no memory leaks in PageMemory
Repository: ignite
Updated Branches:
refs/heads/ignite-4712 [created] 8434edd61
IGNITE-4694 Add tests to check there are no memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3c9e3786
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3c9e3786
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3c9e3786
Branch: refs/heads/ignite-4712
Commit: 3c9e3786a21ec7c8919de9c29bc04d2e3561846c
Parents: c5aee5b
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Wed Feb 15 13:41:08 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Tue Feb 21 12:44:28 2017 +0300
----------------------------------------------------------------------
.../database/IgniteDbAbstractTest.java | 360 +++++++++++++++++++
.../IgniteDbMemoryLeakAbstractTest.java | 84 +++++
.../database/IgniteDbMemoryLeakIndexedTest.java | 85 +++++
.../IgniteDbMemoryLeakLargeObjectsTest.java | 95 +++++
.../IgniteDbMemoryLeakLargePagesTest.java | 90 +++++
.../database/IgniteDbMemoryLeakTest.java | 85 +++++
.../IgniteDbMemoryLeakWithExpirationTest.java | 92 +++++
.../database/IgniteDbPutGetAbstractTest.java | 347 +-----------------
8 files changed, 903 insertions(+), 335 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c9e3786/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
new file mode 100644
index 0000000..3bc7004
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.AffinityFunction;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Random;
+
+/**
+ *
+ */
+public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /**
+ * @return Node count.
+ */
+ protected abstract int gridCount();
+
+ /**
+ * @return {@code True} if indexing is enabled.
+ */
+ protected abstract boolean indexingEnabled();
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ MemoryConfiguration dbCfg = new MemoryConfiguration();
+
+ dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
+
+ if (isLargePage())
+ dbCfg.setPageSize(16 * 1024);
+ else
+ dbCfg.setPageSize(1024);
+
+
+ dbCfg.setPageCacheSize(200 * 1024 * 1024);
+
+ configure(dbCfg);
+
+ cfg.setMemoryConfiguration(dbCfg);
+
+ CacheConfiguration ccfg = new CacheConfiguration();
+
+ if (indexingEnabled())
+ ccfg.setIndexedTypes(Integer.class, DbValue.class);
+
+ ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ CacheConfiguration ccfg2 = new CacheConfiguration("non-primitive");
+
+ if (indexingEnabled())
+ ccfg2.setIndexedTypes(DbKey.class, DbValue.class);
+
+ ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg2.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg2.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ CacheConfiguration ccfg3 = new CacheConfiguration("large");
+
+ if (indexingEnabled())
+ ccfg3.setIndexedTypes(Integer.class, LargeDbValue.class);
+
+ ccfg3.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg3.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg3.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg3.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ CacheConfiguration ccfg4 = new CacheConfiguration("tiny");
+
+ ccfg4.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg4.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg4.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg4.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ final AffinityFunction aff = new RendezvousAffinityFunction(1, null);
+
+ ccfg4.setAffinity(aff);
+
+ cfg.setCacheConfiguration(ccfg, ccfg2, ccfg3, ccfg4);
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(IP_FINDER);
+
+ cfg.setDiscoverySpi(discoSpi);
+ cfg.setMarshaller(null);
+
+ configure(cfg);
+
+ return cfg;
+ }
+
+ protected void configure(IgniteConfiguration cfg){
+ //NOP
+ }
+
+ protected void configure(MemoryConfiguration mCfg){
+ //NOP
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+
+ long seed = 1464583813940L; // System.currentTimeMillis();
+
+ info("Seed: " + seed + "L");
+
+ BPlusTree.rnd = new Random(seed);
+
+ startGrids(gridCount());
+
+ awaitPartitionMapExchange();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ BPlusTree.rnd = null;
+
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ * @return {@code True} if use large page.
+ */
+ protected boolean isLargePage() {
+ return false;
+ }
+
+ /**
+ *
+ */
+ static class DbKey implements Serializable {
+ /** */
+ int val;
+
+ /**
+ * @param val Value.
+ */
+ DbKey(int val) {
+ this.val = val;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || !(o instanceof DbKey))
+ return false;
+
+ DbKey key = (DbKey)o;
+
+ return val == key.val;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return val;
+ }
+ }
+
+ /**
+ *
+ */
+ static class LargeDbKey implements Serializable {
+ /** */
+ int val;
+
+ /** */
+ byte[] data;
+
+ /**
+ * @param val Value.
+ * @param size Key payload size.
+ */
+ LargeDbKey(int val, int size) {
+ this.val = val;
+
+ data = new byte[size];
+
+ Arrays.fill(data, (byte)val);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || !(o instanceof LargeDbKey))
+ return false;
+
+ LargeDbKey key = (LargeDbKey)o;
+
+ return val == key.val && Arrays.equals(data, key.data);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return val + Arrays.hashCode(data);
+ }
+ }
+
+ /**
+ *
+ */
+ static class DbValue implements Serializable {
+ /** */
+ @QuerySqlField(index = true)
+ int iVal;
+
+ /** */
+ @QuerySqlField(index = true)
+ String sVal;
+
+ /** */
+ @QuerySqlField
+ long lVal;
+
+
+
+ /**
+ * @param iVal Integer value.
+ * @param sVal String value.
+ * @param lVal Long value.
+ */
+ DbValue(int iVal, String sVal, long lVal) {
+ this.iVal = iVal;
+ this.sVal = sVal;
+ this.lVal = lVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ DbValue dbVal = (DbValue)o;
+
+ return iVal == dbVal.iVal && lVal == dbVal.lVal &&
+ !(sVal != null ? !sVal.equals(dbVal.sVal) : dbVal.sVal != null);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = iVal;
+
+ res = 31 * res + (sVal != null ? sVal.hashCode() : 0);
+ res = 31 * res + (int)(lVal ^ (lVal >>> 32));
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(DbValue.class, this);
+ }
+ }
+
+ /**
+ *
+ */
+ static class LargeDbValue {
+ /** */
+ @QuerySqlField(index = true)
+ String str1;
+
+ /** */
+ @QuerySqlField(index = true)
+ String str2;
+
+ /** */
+ int[] arr;
+
+ /**
+ * @param str1 String 1.
+ * @param str2 String 2.
+ * @param arr Big array.
+ */
+ LargeDbValue(final String str1, final String str2, final int[] arr) {
+ this.str1 = str1;
+ this.str2 = str2;
+ this.arr = arr;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(final Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ final LargeDbValue that = (LargeDbValue) o;
+
+ if (str1 != null ? !str1.equals(that.str1) : that.str1 != null) return false;
+ if (str2 != null ? !str2.equals(that.str2) : that.str2 != null) return false;
+
+ return Arrays.equals(arr, that.arr);
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = str1 != null ? str1.hashCode() : 0;
+
+ res = 31 * res + (str2 != null ? str2.hashCode() : 0);
+ res = 31 * res + Arrays.hashCode(arr);
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(LargeDbValue.class, this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c9e3786/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
new file mode 100644
index 0000000..6a5d039
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.compute.ComputeTaskFuture;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ */
+public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest {
+
+ /** Test duration in seconds*/
+ protected abstract int duration();
+
+ @Override
+ protected long getTestTimeout() {
+ return duration() * 1200;
+ }
+
+ /** */
+ protected abstract void operation(IgniteEx ig);
+
+ /** */
+ public void testMemoryLeak() throws Exception {
+
+ final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration());
+
+ int tasksCount = Runtime.getRuntime().availableProcessors() * 4;
+
+ IgniteCompute compute = grid(0).compute().withAsync();
+
+ ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCount];
+
+ for (int i = 0; i < tasksCount; i++) {
+ compute.run(new IgniteRunnable() {
+ @IgniteInstanceResource
+ private Ignite ig;
+
+ @Override
+ public void run() {
+ int i = 0;
+ while (System.nanoTime() < end) {
+ operation((IgniteEx) ig);
+
+ if(i++ == 100) {
+ check((IgniteEx) ig);
+ i = 0;
+ }
+ }
+ }
+ });
+
+ futs[i] = compute.future();
+ }
+
+ for (ComputeTaskFuture fut : futs) {
+ fut.get();
+ }
+ }
+
+ protected void check(IgniteEx ig) {}
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c9e3786/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
new file mode 100644
index 0000000..4cd74d0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ *
+ */
+public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakAbstractTest {
+
+ @Override
+ protected int duration() {
+ return 300;
+ }
+
+ @Override
+ protected int gridCount() {
+ return 1;
+ }
+
+ @Override
+ protected void configure(IgniteConfiguration cfg) {
+ cfg.setMetricsLogFrequency(5000);
+ }
+
+ @Override
+ protected void configure(MemoryConfiguration mCfg) {
+ mCfg.setPageCacheSize(1024 * 1024);
+ }
+
+ @Override
+ protected boolean indexingEnabled() {
+ return true;
+ }
+
+ protected void operation(IgniteEx ig){
+ IgniteCache<Object, Object> cache = ig.cache("non-primitive");
+ Random rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < 1000; i++) {
+ DbKey key = new DbKey(rnd.nextInt(200_000));
+
+ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
+
+ switch (rnd.nextInt(3)) {
+ case 0:
+ cache.getAndPut(key, v0);
+ case 1:
+ cache.get(key);
+ break;
+ case 2:
+ cache.getAndRemove(key);
+ }
+ }
+ }
+
+ @Override
+ protected void check(IgniteEx ig) {
+ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+
+ assertTrue(pages < 19100);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c9e3786/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
new file mode 100644
index 0000000..a4d88e1
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ *
+ */
+public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstractTest {
+
+ private final static int[] ARRAY;
+ static {
+ ARRAY = new int[1024];
+ Random rnd = new Random();
+ for (int i = 0; i < ARRAY.length; i++) {
+ ARRAY[i] = rnd.nextInt();
+ }
+
+ }
+
+ @Override
+ protected int duration() {
+ return 300;
+ }
+
+ @Override
+ protected int gridCount() {
+ return 1;
+ }
+
+ @Override
+ protected void configure(IgniteConfiguration cfg) {
+ cfg.setMetricsLogFrequency(5000);
+ }
+
+ @Override
+ protected void configure(MemoryConfiguration mCfg) {
+ mCfg.setPageCacheSize(60 * 1024 * 1024);
+ }
+
+ @Override
+ protected boolean indexingEnabled() {
+ return false;
+ }
+
+ protected void operation(IgniteEx ig){
+ IgniteCache<Object, Object> cache = ig.cache("large");
+ Random rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < 1000; i++) {
+ LargeDbKey key = new LargeDbKey(rnd.nextInt(10_000), 1024);
+
+ LargeDbValue v0 = new LargeDbValue("test-value-1-" + rnd.nextInt(200), "test-value-2-" + rnd.nextInt(200), ARRAY);
+
+ switch (rnd.nextInt(3)) {
+ case 0:
+ cache.getAndPut(key, v0);
+ case 1:
+ cache.get(key);
+ break;
+ case 2:
+ cache.getAndRemove(key);
+ }
+ }
+ }
+
+ @Override
+ protected void check(IgniteEx ig) {
+ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+
+ assertTrue(pages < 50000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c9e3786/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
new file mode 100644
index 0000000..bfa4aa9
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ *
+ */
+public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakAbstractTest {
+
+ @Override
+ protected int duration() {
+ return 300;
+ }
+
+ @Override
+ protected int gridCount() {
+ return 1;
+ }
+
+ @Override
+ protected void configure(IgniteConfiguration cfg) {
+ cfg.setMetricsLogFrequency(5000);
+ }
+
+ @Override
+ protected void configure(MemoryConfiguration mCfg) {
+ mCfg.setPageCacheSize(100 * 1024 * 1024);
+ }
+
+ @Override
+ protected boolean indexingEnabled() {
+ return false;
+ }
+
+ @Override
+ protected boolean isLargePage() {
+ return true;
+ }
+
+ protected void operation(IgniteEx ig){
+ IgniteCache<Object, Object> cache = ig.cache("non-primitive");
+ Random rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < 1000; i++) {
+ DbKey key = new DbKey(rnd.nextInt(200_000));
+
+ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
+
+ switch (rnd.nextInt(3)) {
+ case 0:
+ cache.getAndPut(key, v0);
+ case 1:
+ cache.get(key);
+ break;
+ case 2:
+ cache.getAndRemove(key);
+ }
+ }
+ }
+
+ @Override
+ protected void check(IgniteEx ig) {
+ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+
+ assertTrue(pages < 4600);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c9e3786/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
new file mode 100644
index 0000000..6af4e41
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ *
+ */
+public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
+
+ @Override
+ protected int duration() {
+ return 300;
+ }
+
+ @Override
+ protected int gridCount() {
+ return 1;
+ }
+
+ @Override
+ protected void configure(IgniteConfiguration cfg) {
+ cfg.setMetricsLogFrequency(5000);
+ }
+
+ @Override
+ protected void configure(MemoryConfiguration mCfg) {
+ mCfg.setPageCacheSize(1024 * 1024);
+ }
+
+ @Override
+ protected boolean indexingEnabled() {
+ return false;
+ }
+
+ protected void operation(IgniteEx ig){
+ IgniteCache<Object, Object> cache = ig.cache("non-primitive");
+ Random rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < 1000; i++) {
+ DbKey key = new DbKey(rnd.nextInt(200_000));
+
+ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
+
+ switch (rnd.nextInt(3)) {
+ case 0:
+ cache.getAndPut(key, v0);
+ case 1:
+ cache.get(key);
+ break;
+ case 2:
+ cache.getAndRemove(key);
+ }
+ }
+ }
+
+ @Override
+ protected void check(IgniteEx ig) {
+ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+
+ assertTrue(pages < 19100);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c9e3786/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
new file mode 100644
index 0000000..d9e3f34
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+
+import javax.cache.expiry.CreatedExpiryPolicy;
+import javax.cache.expiry.Duration;
+import javax.cache.expiry.ExpiryPolicy;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+/**
+ *
+ */
+public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakAbstractTest {
+
+ private static final ExpiryPolicy EXPIRY = new CreatedExpiryPolicy(new Duration(MILLISECONDS, 10L));
+
+ @Override
+ protected int duration() {
+ return 300;
+ }
+
+ @Override
+ protected int gridCount() {
+ return 1;
+ }
+
+ @Override
+ protected void configure(IgniteConfiguration cfg) {
+ cfg.setMetricsLogFrequency(5000);
+ }
+
+ @Override
+ protected void configure(MemoryConfiguration mCfg) {
+ mCfg.setPageCacheSize(1024 * 1024);
+ }
+
+ @Override
+ protected boolean indexingEnabled() {
+ return false;
+ }
+
+ protected void operation(IgniteEx ig) {
+ IgniteCache<Object, Object> cache = ig.cache("non-primitive").withExpiryPolicy(EXPIRY);
+ Random rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < 1000; i++) {
+ DbKey key = new DbKey(rnd.nextInt(200_000));
+
+ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
+
+ switch (rnd.nextInt(3)) {
+ case 0:
+ cache.getAndPut(key, v0);
+ case 1:
+ cache.get(key);
+ break;
+ case 2:
+ cache.getAndRemove(key);
+ }
+ }
+ }
+
+ @Override
+ protected void check(IgniteEx ig) {
+ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+
+ assertTrue(pages < 10000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c9e3786/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
index c7a07e3..228a262 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
@@ -17,175 +17,39 @@
package org.apache.ignite.internal.processors.database;
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ThreadLocalRandom;
-import javax.cache.Cache;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CachePeekMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cache.affinity.AffinityFunction;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.cache.query.SqlQuery;
-import org.apache.ignite.cache.query.annotations.QuerySqlField;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
import org.apache.ignite.internal.util.GridRandom;
import org.apache.ignite.internal.util.typedef.PA;
import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.junit.Assert;
+import javax.cache.Cache;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+
/**
*
*/
-public abstract class IgniteDbPutGetAbstractTest extends GridCommonAbstractTest {
- /** */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /**
- * @return Node count.
- */
- protected abstract int gridCount();
-
- /**
- * @return {@code True} if indexing is enabled.
- */
- protected abstract boolean indexingEnabled();
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- MemoryConfiguration dbCfg = new MemoryConfiguration();
-
- if (isLargePage()) {
- dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
-
- dbCfg.setPageSize(16 * 1024);
-
- dbCfg.setPageCacheSize(200 * 1024 * 1024);
- }
- else {
- dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
-
- dbCfg.setPageSize(1024);
-
- dbCfg.setPageCacheSize(200 * 1024 * 1024);
- }
-
- cfg.setMemoryConfiguration(dbCfg);
-
- CacheConfiguration ccfg = new CacheConfiguration();
-
- if (indexingEnabled())
- ccfg.setIndexedTypes(Integer.class, DbValue.class);
-
- ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
- ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- CacheConfiguration ccfg2 = new CacheConfiguration("non-primitive");
-
- if (indexingEnabled())
- ccfg2.setIndexedTypes(DbKey.class, DbValue.class);
-
- ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg2.setRebalanceMode(CacheRebalanceMode.SYNC);
- ccfg2.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- CacheConfiguration ccfg3 = new CacheConfiguration("large");
-
- if (indexingEnabled())
- ccfg3.setIndexedTypes(Integer.class, LargeDbValue.class);
-
- ccfg3.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg3.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg3.setRebalanceMode(CacheRebalanceMode.SYNC);
- ccfg3.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- CacheConfiguration ccfg4 = new CacheConfiguration("tiny");
-
- ccfg4.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg4.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg4.setRebalanceMode(CacheRebalanceMode.SYNC);
- ccfg4.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- final AffinityFunction aff = new RendezvousAffinityFunction(1, null);
-
- ccfg4.setAffinity(aff);
-
- cfg.setCacheConfiguration(ccfg, ccfg2, ccfg3, ccfg4);
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(IP_FINDER);
-
- cfg.setDiscoverySpi(discoSpi);
- cfg.setMarshaller(null);
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
-
- long seed = 1464583813940L; // System.currentTimeMillis();
-
- info("Seed: " + seed + "L");
-
- BPlusTree.rnd = new Random(seed);
-
- startGrids(gridCount());
-
- awaitPartitionMapExchange();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- BPlusTree.rnd = null;
-
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /**
- * @return {@code True} if use large page.
- */
- protected boolean isLargePage() {
- return false;
- };
-
+public abstract class IgniteDbPutGetAbstractTest extends IgniteDbAbstractTest {
/**
*
*/
@@ -1349,191 +1213,4 @@ public abstract class IgniteDbPutGetAbstractTest extends GridCommonAbstractTest
assertNull(internalCache.peekEx(key));
}
-
- /**
- *
- */
- private static class DbKey implements Serializable {
- /** */
- private int val;
-
- /**
- * @param val Value.
- */
- private DbKey(int val) {
- this.val = val;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || !(o instanceof DbKey))
- return false;
-
- DbKey key = (DbKey)o;
-
- return val == key.val;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return val;
- }
- }
-
- /**
- *
- */
- private static class LargeDbKey implements Serializable {
- /** */
- private int val;
-
- /** */
- private byte[] data;
-
- /**
- * @param val Value.
- * @param size Key payload size.
- */
- private LargeDbKey(int val, int size) {
- this.val = val;
-
- data = new byte[size];
-
- Arrays.fill(data, (byte)val);
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || !(o instanceof LargeDbKey))
- return false;
-
- LargeDbKey key = (LargeDbKey)o;
-
- return val == key.val && Arrays.equals(data, key.data);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return val + Arrays.hashCode(data);
- }
- }
-
- /**
- *
- */
- private static class DbValue implements Serializable {
- /** */
- @QuerySqlField(index = true)
- private int iVal;
-
- /** */
- @QuerySqlField(index = true)
- private String sVal;
-
- /** */
- @QuerySqlField
- private long lVal;
-
- /**
- * @param iVal Integer value.
- * @param sVal String value.
- * @param lVal Long value.
- */
- public DbValue(int iVal, String sVal, long lVal) {
- this.iVal = iVal;
- this.sVal = sVal;
- this.lVal = lVal;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- DbValue dbVal = (DbValue)o;
-
- return iVal == dbVal.iVal && lVal == dbVal.lVal &&
- !(sVal != null ? !sVal.equals(dbVal.sVal) : dbVal.sVal != null);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = iVal;
-
- res = 31 * res + (sVal != null ? sVal.hashCode() : 0);
- res = 31 * res + (int)(lVal ^ (lVal >>> 32));
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(DbValue.class, this);
- }
- }
-
- /**
- *
- */
- private static class LargeDbValue {
- /** */
- @QuerySqlField(index = true)
- private String str1;
-
- /** */
- @QuerySqlField(index = true)
- private String str2;
-
- /** */
- private int[] arr;
-
- /**
- * @param str1 String 1.
- * @param str2 String 2.
- * @param arr Big array.
- */
- public LargeDbValue(final String str1, final String str2, final int[] arr) {
- this.str1 = str1;
- this.str2 = str2;
- this.arr = arr;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(final Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- final LargeDbValue that = (LargeDbValue) o;
-
- if (str1 != null ? !str1.equals(that.str1) : that.str1 != null) return false;
- if (str2 != null ? !str2.equals(that.str2) : that.str2 != null) return false;
-
- return Arrays.equals(arr, that.arr);
-
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = str1 != null ? str1.hashCode() : 0;
-
- res = 31 * res + (str2 != null ? str2.hashCode() : 0);
- res = 31 * res + Arrays.hashCode(arr);
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(LargeDbValue.class, this);
- }
- }
}
[06/43] ignite git commit: ignite-4694 review
Posted by sb...@apache.org.
ignite-4694 review
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8e12097f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8e12097f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8e12097f
Branch: refs/heads/ignite-4712
Commit: 8e12097f9094d7f155135ee2f4c9c33f5f7af9aa
Parents: 3c9e378
Author: sboikov <sb...@gridgain.com>
Authored: Wed Feb 15 15:08:14 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Tue Feb 21 12:44:29 2017 +0300
----------------------------------------------------------------------
.../database/IgniteDbMemoryLeakAbstractTest.java | 15 ++++++++-------
.../database/IgniteDbMemoryLeakIndexedTest.java | 3 +++
.../database/IgniteDbMemoryLeakLargeObjectsTest.java | 9 +++++----
.../database/IgniteDbMemoryLeakLargePagesTest.java | 2 ++
.../processors/database/IgniteDbMemoryLeakTest.java | 10 +++++++---
.../IgniteDbMemoryLeakWithExpirationTest.java | 2 +-
6 files changed, 26 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8e12097f/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index 6a5d039..fc0e715 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -27,10 +27,10 @@ import org.apache.ignite.resources.IgniteInstanceResource;
import java.util.concurrent.TimeUnit;
/**
- *
+ * TODO: fix javadoc warnings, code style.
*/
public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest {
-
+ // TODO: take duration from system property.
/** Test duration in seconds*/
protected abstract int duration();
@@ -44,16 +44,18 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/** */
public void testMemoryLeak() throws Exception {
+ // TODO: take PageMemory max size is the same as we configured.
final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration());
- int tasksCount = Runtime.getRuntime().availableProcessors() * 4;
+ // TODO: use threads instead of compute or make sure there are enough threads in pool.
+ int tasksCnt = Runtime.getRuntime().availableProcessors() * 4;
IgniteCompute compute = grid(0).compute().withAsync();
- ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCount];
+ ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCnt];
- for (int i = 0; i < tasksCount; i++) {
+ for (int i = 0; i < tasksCnt; i++) {
compute.run(new IgniteRunnable() {
@IgniteInstanceResource
private Ignite ig;
@@ -75,9 +77,8 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
futs[i] = compute.future();
}
- for (ComputeTaskFuture fut : futs) {
+ for (ComputeTaskFuture fut : futs)
fut.get();
- }
}
protected void check(IgniteEx ig) {}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8e12097f/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
index 4cd74d0..db77131 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
@@ -51,6 +51,7 @@ public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakAbstractTes
}
@Override
+ // TODO: move test to module ignite-indexing.
protected boolean indexingEnabled() {
return true;
}
@@ -64,6 +65,8 @@ public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakAbstractTes
DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
+ // TODO: also execute sql queries.
+
switch (rnd.nextInt(3)) {
case 0:
cache.getAndPut(key, v0);
http://git-wip-us.apache.org/repos/asf/ignite/blob/8e12097f/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
index a4d88e1..2a6c8cd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
@@ -29,15 +29,16 @@ import java.util.concurrent.ThreadLocalRandom;
*
*/
public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstractTest {
-
+ /** */
private final static int[] ARRAY;
+
static {
ARRAY = new int[1024];
+
Random rnd = new Random();
- for (int i = 0; i < ARRAY.length; i++) {
- ARRAY[i] = rnd.nextInt();
- }
+ for (int i = 0; i < ARRAY.length; i++)
+ ARRAY[i] = rnd.nextInt();
}
@Override
http://git-wip-us.apache.org/repos/asf/ignite/blob/8e12097f/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
index bfa4aa9..91c96af 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
@@ -47,6 +47,7 @@ public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakAbstract
@Override
protected void configure(MemoryConfiguration mCfg) {
+ // TODO: understand why such overhead with large pages.
mCfg.setPageCacheSize(100 * 1024 * 1024);
}
@@ -60,6 +61,7 @@ public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakAbstract
return true;
}
+ // TODO: avoid copy/paste.
protected void operation(IgniteEx ig){
IgniteCache<Object, Object> cache = ig.cache("non-primitive");
Random rnd = ThreadLocalRandom.current();
http://git-wip-us.apache.org/repos/asf/ignite/blob/8e12097f/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
index 6af4e41..2b0ce1e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
@@ -29,9 +29,8 @@ import java.util.concurrent.ThreadLocalRandom;
*
*/
public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
-
- @Override
- protected int duration() {
+ /** {@inheritDoc} */
+ @Override protected int duration() {
return 300;
}
@@ -64,12 +63,17 @@ public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
+ // TODO: also execute scan query.
+
switch (rnd.nextInt(3)) {
case 0:
cache.getAndPut(key, v0);
+ break;
+
case 1:
cache.get(key);
break;
+
case 2:
cache.getAndRemove(key);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8e12097f/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
index d9e3f34..95fe8c8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
@@ -34,7 +34,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS;
*
*/
public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakAbstractTest {
-
+ /** */
private static final ExpiryPolicy EXPIRY = new CreatedExpiryPolicy(new Duration(MILLISECONDS, 10L));
@Override
[40/43] ignite git commit: IGNITE-4694 Add tests to check there are
no memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4694 Add tests to check there are no memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b2f1ea99
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b2f1ea99
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b2f1ea99
Branch: refs/heads/ignite-4712
Commit: b2f1ea996669022bf565aa6aa669a594f632f9c9
Parents: 27178f1
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Wed Feb 22 15:01:09 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:55 2017 +0300
----------------------------------------------------------------------
.../processors/database/IgniteDbMemoryLeakAbstractTest.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b2f1ea99/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index 04ac5ea..06ecb94 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -25,6 +25,8 @@ import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.processors.cache.database.DataStructure;
+import static org.apache.ignite.IgniteSystemProperties.getInteger;
+
/**
* Base class for memory leaks tests.
*/
@@ -84,7 +86,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
* @return Test duration in seconds.
*/
protected int duration() {
- return 300;
+ return getInteger("IGNITE_MEMORY_LEAKS_TEST_DURATION", 300);
}
/**
@@ -92,7 +94,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
*/
@SuppressWarnings("WeakerAccess")
protected int warmUp() {
- return 450;
+ return getInteger("IGNITE_MEMORY_LEAKS_TEST_WARM_UP", 450);
}
/** {@inheritDoc} */
[09/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e259b7a2
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e259b7a2
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e259b7a2
Branch: refs/heads/ignite-4712
Commit: e259b7a2032de9ec1334668e10f92fbae9e5c096
Parents: 84c03e0
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Mon Feb 20 13:40:05 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Tue Feb 21 12:44:29 2017 +0300
----------------------------------------------------------------------
.../cache/database/freelist/PagesList.java | 39 ++++++++++++--------
.../IgniteDbMemoryLeakLargeObjectsTest.java | 14 ++++---
2 files changed, 32 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e259b7a2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
index e5430cf..c03f432 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
@@ -76,9 +76,6 @@ public abstract class PagesList extends DataStructure {
Math.min(8, Runtime.getRuntime().availableProcessors() * 2));
/** */
- private final boolean trackBucketsSize = IgniteSystemProperties.getBoolean("IGNITE_PAGES_LIST_TRACK_SIZE", false);
-
- /** */
protected final LongAdder8[] bucketsSize;
/** Page ID to store list metadata. */
@@ -600,8 +597,7 @@ public abstract class PagesList extends DataStructure {
if (idx == -1)
handlePageFull(pageId, page, pageAddr, io, dataPage, dataPageAddr, bucket);
else {
- if (trackBucketsSize)
- bucketsSize[bucket].increment();
+ bucketsSize[bucket].increment();
if (isWalDeltaRecordNeeded(wal, page))
wal.log(new PagesListAddPageRecord(cacheId, pageId, dataPageId));
@@ -695,8 +691,7 @@ public abstract class PagesList extends DataStructure {
assert idx != -1;
- if (trackBucketsSize)
- bucketsSize[bucket].increment();
+ bucketsSize[bucket].increment();
dataIO.setFreeListPageId(dataPageAddr, nextId);
@@ -786,8 +781,7 @@ public abstract class PagesList extends DataStructure {
}
}
else {
- if (trackBucketsSize)
- bucketsSize[bucket].increment();
+ bucketsSize[bucket].increment();
// TODO: use single WAL record for bag?
if (isWalDeltaRecordNeeded(wal, page))
@@ -819,7 +813,18 @@ public abstract class PagesList extends DataStructure {
if (tails == null)
return null;
- return randomTail(tails);
+ int len = tails.length;
+ int init = randomInt(len);
+ int cur = init;
+
+ while (true) {
+ Stripe stripe = tails[cur];
+ if(!stripe.empty)
+ return stripe;
+
+ if((cur = (cur + 1) % len) == init)
+ return null;
+ }
}
/**
@@ -873,7 +878,7 @@ public abstract class PagesList extends DataStructure {
for (int lockAttempt = 0; ;) {
Stripe stripe = getPageForTake(bucket);
- if (stripe == null || stripe.empty)
+ if (stripe == null)
return 0L;
long tailId = stripe.tailId;
@@ -904,8 +909,7 @@ public abstract class PagesList extends DataStructure {
long pageId = io.takeAnyPage(tailPageAddr);
if (pageId != 0L) {
- if (trackBucketsSize)
- bucketsSize[bucket].decrement();
+ bucketsSize[bucket].decrement();
if (isWalDeltaRecordNeeded(wal, tail))
wal.log(new PagesListRemovePageRecord(cacheId, tailId, pageId));
@@ -965,8 +969,12 @@ public abstract class PagesList extends DataStructure {
ret = tailId;
}
- else
+ else {
stripe.empty = true;
+
+ if (bucketsSize[bucket].sum() > 0)
+ continue;
+ }
}
// If we do not have a previous page (we are at head), then we still can return
@@ -1026,8 +1034,7 @@ public abstract class PagesList extends DataStructure {
if (!rmvd)
return false;
- if (trackBucketsSize)
- bucketsSize[bucket].decrement();
+ bucketsSize[bucket].decrement();
if (isWalDeltaRecordNeeded(wal, page))
wal.log(new PagesListRemovePageRecord(cacheId, pageId, dataPageId));
http://git-wip-us.apache.org/repos/asf/ignite/blob/e259b7a2/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
index c1b1c9b..0fa9096 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
/**
@@ -35,6 +36,13 @@ public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstra
}
/** {@inheritDoc} */
+ @Override protected void configure(MemoryConfiguration mCfg) {
+ super.configure(mCfg);
+
+ mCfg.setPageCacheSize(35840000); // The space for 35000 pages
+ }
+
+ /** {@inheritDoc} */
@Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
return ig.cache("large");
}
@@ -49,12 +57,8 @@ public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstra
return new LargeDbValue("test-value-1-" + getRandom().nextInt(200), "test-value-2-" + getRandom().nextInt(200), ARRAY);
}
- @Override protected int warmUp() {
- return 600;
- }
-
/** {@inheritDoc} */
@Override protected long pagesMax() {
- return 2000000;
+ return 35000;
}
}
[42/43] ignite git commit: Merge remote-tracking branch
'remotes/origin/ignite-3477' into ignite-4712
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/ignite-3477' into ignite-4712
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/40dbfbd0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/40dbfbd0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/40dbfbd0
Branch: refs/heads/ignite-4712
Commit: 40dbfbd0874e33614568f2f942c358ca87bc1152
Parents: d275d5d 13c3cfc
Author: sboikov <sb...@gridgain.com>
Authored: Fri Mar 10 10:03:44 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Mar 10 10:03:44 2017 +0300
----------------------------------------------------------------------
.../store/jdbc/CacheAbstractJdbcStore.java | 15 +-
.../configuration/CacheConfiguration.java | 28 ++
.../cache/IgniteCacheOffheapManagerImpl.java | 33 +-
.../cache/database/tree/BPlusTree.java | 18 +
.../cache/database/tree/io/BPlusIO.java | 9 +
.../processors/query/h2/database/H2Tree.java | 120 +++++-
.../query/h2/database/H2TreeIndex.java | 167 ++------
.../query/h2/database/InlineIndexHelper.java | 400 ++++++++++++++++---
.../query/h2/database/io/H2ExtrasInnerIO.java | 12 +-
.../query/h2/database/io/H2ExtrasLeafIO.java | 9 +-
.../processors/query/h2/opt/GridH2Table.java | 4 +-
.../cache/ttl/CacheTtlAbstractSelfTest.java | 16 +-
.../query/h2/database/H2TreeIndexTest.java | 59 ---
.../h2/database/InlineIndexHelperTest.java | 316 ++++++++++++++-
.../yardstick/IgniteAbstractBenchmark.java | 6 +
.../yardstick/IgniteBenchmarkArguments.java | 11 +
.../yardstick/cache/IgnitePutAllBenchmark.java | 95 ++++-
.../cache/IgnitePutAllTxBenchmark.java | 43 +-
18 files changed, 1012 insertions(+), 349 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/40dbfbd0/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
----------------------------------------------------------------------
[41/43] ignite git commit: Merge remote-tracking branch
'community/ignite-4712' into ignite-4712
Posted by sb...@apache.org.
Merge remote-tracking branch 'community/ignite-4712' into ignite-4712
# Conflicts:
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d275d5d5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d275d5d5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d275d5d5
Branch: refs/heads/ignite-4712
Commit: d275d5d558effd1371ec2371cef74b1e493934ec
Parents: c826a97 869089d
Author: sboikov <sb...@gridgain.com>
Authored: Fri Mar 10 10:03:24 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Mar 10 10:03:24 2017 +0300
----------------------------------------------------------------------
.../benchmarks/jmh/tree/BPlusTreeBenchmark.java | 3 +-
.../discovery/GridDiscoveryManager.java | 11 +
.../internal/pagemem/wal/record/DataRecord.java | 10 +-
.../processors/cache/GridCacheMapEntry.java | 1729 +++++++++++-------
.../cache/GridCacheUpdateAtomicResult.java | 96 +-
.../cache/IgniteCacheOffheapManager.java | 48 +
.../cache/IgniteCacheOffheapManagerImpl.java | 236 ++-
.../processors/cache/database/CacheDataRow.java | 6 +
.../cache/database/CacheDataRowAdapter.java | 13 +
.../cache/database/MetadataStorage.java | 2 +-
.../cache/database/tree/BPlusTree.java | 936 ++++++++--
.../distributed/dht/GridDhtCacheEntry.java | 5 +
.../query/GridCacheDistributedQueryFuture.java | 7 +-
.../utils/PlatformConfigurationUtils.java | 18 -
.../apache/ignite/internal/util/IgniteTree.java | 47 +
.../tcp/internal/TcpDiscoveryNodesRing.java | 3 +-
...eDynamicCacheStartNoExchangeTimeoutTest.java | 8 +-
.../processors/database/BPlusTreeSelfTest.java | 272 ++-
.../database/FreeListImplSelfTest.java | 5 +
.../platform/PlatformEventsWriteEventTask.java | 7 +-
.../junits/common/GridCommonAbstractTest.java | 2 +-
.../processors/query/h2/database/H2Tree.java | 2 +-
.../processors/query/h2/opt/GridH2Row.java | 5 +
.../processors/query/h2/opt/GridH2Table.java | 10 +-
.../query/h2/opt/GridH2TreeIndex.java | 5 +
.../Apache.Ignite.Core.Tests.csproj | 1 -
.../Cache/CacheAbstractTest.cs | 17 +-
.../Cache/CacheConfigurationTest.cs | 8 -
.../Cache/CacheSwapSpaceTest.cs | 122 --
.../Cache/Store/CacheStoreTest.cs | 24 +-
.../Config/native-client-test-cache.xml | 5 -
.../Apache.Ignite.Core.Tests/DeploymentTest.cs | 1 -
.../Apache.Ignite.Core.Tests/EventsTest.cs | 37 +-
.../IgniteConfigurationSerializerTest.cs | 20 -
.../IgniteConfigurationTest.cs | 20 +-
.../Apache.Ignite.Core.csproj | 5 -
.../Apache.Ignite.Core/Cache/CachePeekMode.cs | 7 +-
.../Cache/Configuration/CacheConfiguration.cs | 26 -
.../Cache/Configuration/CacheMemoryMode.cs | 60 -
.../Apache.Ignite.Core/Events/EventReader.cs | 1 -
.../Apache.Ignite.Core/Events/EventType.cs | 57 -
.../Apache.Ignite.Core/Events/SwapSpaceEvent.cs | 51 -
.../Apache.Ignite.Core/IgniteConfiguration.cs | 15 +-
.../IgniteConfigurationSection.xsd | 55 -
.../Binary/BinaryEqualityComparerSerializer.cs | 2 +-
.../Impl/SwapSpace/SwapSpaceSerializer.cs | 99 -
.../SwapSpace/File/FileSwapSpaceSpi.cs | 105 --
.../SwapSpace/ISwapSpaceSpi.cs | 35 -
.../Datagrid/MultiTieredCacheExample.cs | 24 +-
49 files changed, 2543 insertions(+), 1740 deletions(-)
----------------------------------------------------------------------
[17/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d89e0e69
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d89e0e69
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d89e0e69
Branch: refs/heads/ignite-4712
Commit: d89e0e69644d5f8ed7d8aa838f180444616fa712
Parents: 1a2bbef
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Wed Feb 22 12:28:30 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Feb 22 12:28:30 2017 +0300
----------------------------------------------------------------------
.../internal/processors/database/IgniteDbPutGetAbstractTest.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d89e0e69/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
index 228a262..cf98dce 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
@@ -60,9 +60,7 @@ public abstract class IgniteDbPutGetAbstractTest extends IgniteDbAbstractTest {
final int cnt = 100_000;
- Random rnd = BPlusTree.rnd;
-
- assert rnd != null;
+ Random rnd = new Random();
Map<Integer, DbValue> map = new HashMap<>();
[25/43] ignite git commit: ignite-4694 review
Posted by sb...@apache.org.
ignite-4694 review
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/88c04bb8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/88c04bb8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/88c04bb8
Branch: refs/heads/ignite-4712
Commit: 88c04bb87786ac90b400872244afb29db6e77e5d
Parents: ef8e479
Author: sboikov <sb...@gridgain.com>
Authored: Wed Feb 15 15:08:14 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:54 2017 +0300
----------------------------------------------------------------------
.../database/IgniteDbMemoryLeakAbstractTest.java | 15 ++++++++-------
.../database/IgniteDbMemoryLeakIndexedTest.java | 3 +++
.../database/IgniteDbMemoryLeakLargeObjectsTest.java | 9 +++++----
.../database/IgniteDbMemoryLeakLargePagesTest.java | 2 ++
.../processors/database/IgniteDbMemoryLeakTest.java | 10 +++++++---
.../IgniteDbMemoryLeakWithExpirationTest.java | 2 +-
6 files changed, 26 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/88c04bb8/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index 6a5d039..fc0e715 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -27,10 +27,10 @@ import org.apache.ignite.resources.IgniteInstanceResource;
import java.util.concurrent.TimeUnit;
/**
- *
+ * TODO: fix javadoc warnings, code style.
*/
public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest {
-
+ // TODO: take duration from system property.
/** Test duration in seconds*/
protected abstract int duration();
@@ -44,16 +44,18 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/** */
public void testMemoryLeak() throws Exception {
+ // TODO: take PageMemory max size is the same as we configured.
final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration());
- int tasksCount = Runtime.getRuntime().availableProcessors() * 4;
+ // TODO: use threads instead of compute or make sure there are enough threads in pool.
+ int tasksCnt = Runtime.getRuntime().availableProcessors() * 4;
IgniteCompute compute = grid(0).compute().withAsync();
- ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCount];
+ ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCnt];
- for (int i = 0; i < tasksCount; i++) {
+ for (int i = 0; i < tasksCnt; i++) {
compute.run(new IgniteRunnable() {
@IgniteInstanceResource
private Ignite ig;
@@ -75,9 +77,8 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
futs[i] = compute.future();
}
- for (ComputeTaskFuture fut : futs) {
+ for (ComputeTaskFuture fut : futs)
fut.get();
- }
}
protected void check(IgniteEx ig) {}
http://git-wip-us.apache.org/repos/asf/ignite/blob/88c04bb8/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
index 4cd74d0..db77131 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
@@ -51,6 +51,7 @@ public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakAbstractTes
}
@Override
+ // TODO: move test to module ignite-indexing.
protected boolean indexingEnabled() {
return true;
}
@@ -64,6 +65,8 @@ public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakAbstractTes
DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
+ // TODO: also execute sql queries.
+
switch (rnd.nextInt(3)) {
case 0:
cache.getAndPut(key, v0);
http://git-wip-us.apache.org/repos/asf/ignite/blob/88c04bb8/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
index a4d88e1..2a6c8cd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
@@ -29,15 +29,16 @@ import java.util.concurrent.ThreadLocalRandom;
*
*/
public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstractTest {
-
+ /** */
private final static int[] ARRAY;
+
static {
ARRAY = new int[1024];
+
Random rnd = new Random();
- for (int i = 0; i < ARRAY.length; i++) {
- ARRAY[i] = rnd.nextInt();
- }
+ for (int i = 0; i < ARRAY.length; i++)
+ ARRAY[i] = rnd.nextInt();
}
@Override
http://git-wip-us.apache.org/repos/asf/ignite/blob/88c04bb8/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
index bfa4aa9..91c96af 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
@@ -47,6 +47,7 @@ public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakAbstract
@Override
protected void configure(MemoryConfiguration mCfg) {
+ // TODO: understand why such overhead with large pages.
mCfg.setPageCacheSize(100 * 1024 * 1024);
}
@@ -60,6 +61,7 @@ public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakAbstract
return true;
}
+ // TODO: avoid copy/paste.
protected void operation(IgniteEx ig){
IgniteCache<Object, Object> cache = ig.cache("non-primitive");
Random rnd = ThreadLocalRandom.current();
http://git-wip-us.apache.org/repos/asf/ignite/blob/88c04bb8/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
index 6af4e41..2b0ce1e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
@@ -29,9 +29,8 @@ import java.util.concurrent.ThreadLocalRandom;
*
*/
public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
-
- @Override
- protected int duration() {
+ /** {@inheritDoc} */
+ @Override protected int duration() {
return 300;
}
@@ -64,12 +63,17 @@ public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
+ // TODO: also execute scan query.
+
switch (rnd.nextInt(3)) {
case 0:
cache.getAndPut(key, v0);
+ break;
+
case 1:
cache.get(key);
break;
+
case 2:
cache.getAndRemove(key);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/88c04bb8/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
index d9e3f34..95fe8c8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
@@ -34,7 +34,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS;
*
*/
public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakAbstractTest {
-
+ /** */
private static final ExpiryPolicy EXPIRY = new CreatedExpiryPolicy(new Duration(MILLISECONDS, 10L));
@Override
[21/43] ignite git commit: IGNITE-4694 Add tests to check there are
no memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4694 Add tests to check there are no memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ef8e4799
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ef8e4799
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ef8e4799
Branch: refs/heads/ignite-4712
Commit: ef8e4799ab20f5451494ed660abb77d8fe645a5b
Parents: ee28b9c
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Wed Feb 15 13:41:08 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:53 2017 +0300
----------------------------------------------------------------------
.../database/IgniteDbAbstractTest.java | 360 +++++++++++++++++++
.../IgniteDbMemoryLeakAbstractTest.java | 84 +++++
.../database/IgniteDbMemoryLeakIndexedTest.java | 85 +++++
.../IgniteDbMemoryLeakLargeObjectsTest.java | 95 +++++
.../IgniteDbMemoryLeakLargePagesTest.java | 90 +++++
.../database/IgniteDbMemoryLeakTest.java | 85 +++++
.../IgniteDbMemoryLeakWithExpirationTest.java | 92 +++++
.../database/IgniteDbPutGetAbstractTest.java | 347 +-----------------
8 files changed, 903 insertions(+), 335 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ef8e4799/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
new file mode 100644
index 0000000..3bc7004
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.AffinityFunction;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Random;
+
+/**
+ *
+ */
+public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /**
+ * @return Node count.
+ */
+ protected abstract int gridCount();
+
+ /**
+ * @return {@code True} if indexing is enabled.
+ */
+ protected abstract boolean indexingEnabled();
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ MemoryConfiguration dbCfg = new MemoryConfiguration();
+
+ dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
+
+ if (isLargePage())
+ dbCfg.setPageSize(16 * 1024);
+ else
+ dbCfg.setPageSize(1024);
+
+
+ dbCfg.setPageCacheSize(200 * 1024 * 1024);
+
+ configure(dbCfg);
+
+ cfg.setMemoryConfiguration(dbCfg);
+
+ CacheConfiguration ccfg = new CacheConfiguration();
+
+ if (indexingEnabled())
+ ccfg.setIndexedTypes(Integer.class, DbValue.class);
+
+ ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ CacheConfiguration ccfg2 = new CacheConfiguration("non-primitive");
+
+ if (indexingEnabled())
+ ccfg2.setIndexedTypes(DbKey.class, DbValue.class);
+
+ ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg2.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg2.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ CacheConfiguration ccfg3 = new CacheConfiguration("large");
+
+ if (indexingEnabled())
+ ccfg3.setIndexedTypes(Integer.class, LargeDbValue.class);
+
+ ccfg3.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg3.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg3.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg3.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ CacheConfiguration ccfg4 = new CacheConfiguration("tiny");
+
+ ccfg4.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg4.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg4.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg4.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ final AffinityFunction aff = new RendezvousAffinityFunction(1, null);
+
+ ccfg4.setAffinity(aff);
+
+ cfg.setCacheConfiguration(ccfg, ccfg2, ccfg3, ccfg4);
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(IP_FINDER);
+
+ cfg.setDiscoverySpi(discoSpi);
+ cfg.setMarshaller(null);
+
+ configure(cfg);
+
+ return cfg;
+ }
+
+ protected void configure(IgniteConfiguration cfg){
+ //NOP
+ }
+
+ protected void configure(MemoryConfiguration mCfg){
+ //NOP
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+
+ long seed = 1464583813940L; // System.currentTimeMillis();
+
+ info("Seed: " + seed + "L");
+
+ BPlusTree.rnd = new Random(seed);
+
+ startGrids(gridCount());
+
+ awaitPartitionMapExchange();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ BPlusTree.rnd = null;
+
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ * @return {@code True} if use large page.
+ */
+ protected boolean isLargePage() {
+ return false;
+ }
+
+ /**
+ *
+ */
+ static class DbKey implements Serializable {
+ /** */
+ int val;
+
+ /**
+ * @param val Value.
+ */
+ DbKey(int val) {
+ this.val = val;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || !(o instanceof DbKey))
+ return false;
+
+ DbKey key = (DbKey)o;
+
+ return val == key.val;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return val;
+ }
+ }
+
+ /**
+ *
+ */
+ static class LargeDbKey implements Serializable {
+ /** */
+ int val;
+
+ /** */
+ byte[] data;
+
+ /**
+ * @param val Value.
+ * @param size Key payload size.
+ */
+ LargeDbKey(int val, int size) {
+ this.val = val;
+
+ data = new byte[size];
+
+ Arrays.fill(data, (byte)val);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || !(o instanceof LargeDbKey))
+ return false;
+
+ LargeDbKey key = (LargeDbKey)o;
+
+ return val == key.val && Arrays.equals(data, key.data);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return val + Arrays.hashCode(data);
+ }
+ }
+
+ /**
+ *
+ */
+ static class DbValue implements Serializable {
+ /** */
+ @QuerySqlField(index = true)
+ int iVal;
+
+ /** */
+ @QuerySqlField(index = true)
+ String sVal;
+
+ /** */
+ @QuerySqlField
+ long lVal;
+
+
+
+ /**
+ * @param iVal Integer value.
+ * @param sVal String value.
+ * @param lVal Long value.
+ */
+ DbValue(int iVal, String sVal, long lVal) {
+ this.iVal = iVal;
+ this.sVal = sVal;
+ this.lVal = lVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ DbValue dbVal = (DbValue)o;
+
+ return iVal == dbVal.iVal && lVal == dbVal.lVal &&
+ !(sVal != null ? !sVal.equals(dbVal.sVal) : dbVal.sVal != null);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = iVal;
+
+ res = 31 * res + (sVal != null ? sVal.hashCode() : 0);
+ res = 31 * res + (int)(lVal ^ (lVal >>> 32));
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(DbValue.class, this);
+ }
+ }
+
+ /**
+ *
+ */
+ static class LargeDbValue {
+ /** */
+ @QuerySqlField(index = true)
+ String str1;
+
+ /** */
+ @QuerySqlField(index = true)
+ String str2;
+
+ /** */
+ int[] arr;
+
+ /**
+ * @param str1 String 1.
+ * @param str2 String 2.
+ * @param arr Big array.
+ */
+ LargeDbValue(final String str1, final String str2, final int[] arr) {
+ this.str1 = str1;
+ this.str2 = str2;
+ this.arr = arr;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(final Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ final LargeDbValue that = (LargeDbValue) o;
+
+ if (str1 != null ? !str1.equals(that.str1) : that.str1 != null) return false;
+ if (str2 != null ? !str2.equals(that.str2) : that.str2 != null) return false;
+
+ return Arrays.equals(arr, that.arr);
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = str1 != null ? str1.hashCode() : 0;
+
+ res = 31 * res + (str2 != null ? str2.hashCode() : 0);
+ res = 31 * res + Arrays.hashCode(arr);
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(LargeDbValue.class, this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ef8e4799/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
new file mode 100644
index 0000000..6a5d039
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.compute.ComputeTaskFuture;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ */
+public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest {
+
+ /** Test duration in seconds*/
+ protected abstract int duration();
+
+ @Override
+ protected long getTestTimeout() {
+ return duration() * 1200;
+ }
+
+ /** */
+ protected abstract void operation(IgniteEx ig);
+
+ /** */
+ public void testMemoryLeak() throws Exception {
+
+ final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration());
+
+ int tasksCount = Runtime.getRuntime().availableProcessors() * 4;
+
+ IgniteCompute compute = grid(0).compute().withAsync();
+
+ ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCount];
+
+ for (int i = 0; i < tasksCount; i++) {
+ compute.run(new IgniteRunnable() {
+ @IgniteInstanceResource
+ private Ignite ig;
+
+ @Override
+ public void run() {
+ int i = 0;
+ while (System.nanoTime() < end) {
+ operation((IgniteEx) ig);
+
+ if(i++ == 100) {
+ check((IgniteEx) ig);
+ i = 0;
+ }
+ }
+ }
+ });
+
+ futs[i] = compute.future();
+ }
+
+ for (ComputeTaskFuture fut : futs) {
+ fut.get();
+ }
+ }
+
+ protected void check(IgniteEx ig) {}
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ef8e4799/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
new file mode 100644
index 0000000..4cd74d0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ *
+ */
+public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakAbstractTest {
+
+ @Override
+ protected int duration() {
+ return 300;
+ }
+
+ @Override
+ protected int gridCount() {
+ return 1;
+ }
+
+ @Override
+ protected void configure(IgniteConfiguration cfg) {
+ cfg.setMetricsLogFrequency(5000);
+ }
+
+ @Override
+ protected void configure(MemoryConfiguration mCfg) {
+ mCfg.setPageCacheSize(1024 * 1024);
+ }
+
+ @Override
+ protected boolean indexingEnabled() {
+ return true;
+ }
+
+ protected void operation(IgniteEx ig){
+ IgniteCache<Object, Object> cache = ig.cache("non-primitive");
+ Random rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < 1000; i++) {
+ DbKey key = new DbKey(rnd.nextInt(200_000));
+
+ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
+
+ switch (rnd.nextInt(3)) {
+ case 0:
+ cache.getAndPut(key, v0);
+ case 1:
+ cache.get(key);
+ break;
+ case 2:
+ cache.getAndRemove(key);
+ }
+ }
+ }
+
+ @Override
+ protected void check(IgniteEx ig) {
+ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+
+ assertTrue(pages < 19100);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ef8e4799/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
new file mode 100644
index 0000000..a4d88e1
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ *
+ */
+public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstractTest {
+
+ private final static int[] ARRAY;
+ static {
+ ARRAY = new int[1024];
+ Random rnd = new Random();
+ for (int i = 0; i < ARRAY.length; i++) {
+ ARRAY[i] = rnd.nextInt();
+ }
+
+ }
+
+ @Override
+ protected int duration() {
+ return 300;
+ }
+
+ @Override
+ protected int gridCount() {
+ return 1;
+ }
+
+ @Override
+ protected void configure(IgniteConfiguration cfg) {
+ cfg.setMetricsLogFrequency(5000);
+ }
+
+ @Override
+ protected void configure(MemoryConfiguration mCfg) {
+ mCfg.setPageCacheSize(60 * 1024 * 1024);
+ }
+
+ @Override
+ protected boolean indexingEnabled() {
+ return false;
+ }
+
+ protected void operation(IgniteEx ig){
+ IgniteCache<Object, Object> cache = ig.cache("large");
+ Random rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < 1000; i++) {
+ LargeDbKey key = new LargeDbKey(rnd.nextInt(10_000), 1024);
+
+ LargeDbValue v0 = new LargeDbValue("test-value-1-" + rnd.nextInt(200), "test-value-2-" + rnd.nextInt(200), ARRAY);
+
+ switch (rnd.nextInt(3)) {
+ case 0:
+ cache.getAndPut(key, v0);
+ case 1:
+ cache.get(key);
+ break;
+ case 2:
+ cache.getAndRemove(key);
+ }
+ }
+ }
+
+ @Override
+ protected void check(IgniteEx ig) {
+ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+
+ assertTrue(pages < 50000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ef8e4799/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
new file mode 100644
index 0000000..bfa4aa9
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ *
+ */
+public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakAbstractTest {
+
+ @Override
+ protected int duration() {
+ return 300;
+ }
+
+ @Override
+ protected int gridCount() {
+ return 1;
+ }
+
+ @Override
+ protected void configure(IgniteConfiguration cfg) {
+ cfg.setMetricsLogFrequency(5000);
+ }
+
+ @Override
+ protected void configure(MemoryConfiguration mCfg) {
+ mCfg.setPageCacheSize(100 * 1024 * 1024);
+ }
+
+ @Override
+ protected boolean indexingEnabled() {
+ return false;
+ }
+
+ @Override
+ protected boolean isLargePage() {
+ return true;
+ }
+
+ protected void operation(IgniteEx ig){
+ IgniteCache<Object, Object> cache = ig.cache("non-primitive");
+ Random rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < 1000; i++) {
+ DbKey key = new DbKey(rnd.nextInt(200_000));
+
+ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
+
+ switch (rnd.nextInt(3)) {
+ case 0:
+ cache.getAndPut(key, v0);
+ case 1:
+ cache.get(key);
+ break;
+ case 2:
+ cache.getAndRemove(key);
+ }
+ }
+ }
+
+ @Override
+ protected void check(IgniteEx ig) {
+ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+
+ assertTrue(pages < 4600);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ef8e4799/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
new file mode 100644
index 0000000..6af4e41
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ *
+ */
+public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
+
+ @Override
+ protected int duration() {
+ return 300;
+ }
+
+ @Override
+ protected int gridCount() {
+ return 1;
+ }
+
+ @Override
+ protected void configure(IgniteConfiguration cfg) {
+ cfg.setMetricsLogFrequency(5000);
+ }
+
+ @Override
+ protected void configure(MemoryConfiguration mCfg) {
+ mCfg.setPageCacheSize(1024 * 1024);
+ }
+
+ @Override
+ protected boolean indexingEnabled() {
+ return false;
+ }
+
+ protected void operation(IgniteEx ig){
+ IgniteCache<Object, Object> cache = ig.cache("non-primitive");
+ Random rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < 1000; i++) {
+ DbKey key = new DbKey(rnd.nextInt(200_000));
+
+ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
+
+ switch (rnd.nextInt(3)) {
+ case 0:
+ cache.getAndPut(key, v0);
+ case 1:
+ cache.get(key);
+ break;
+ case 2:
+ cache.getAndRemove(key);
+ }
+ }
+ }
+
+ @Override
+ protected void check(IgniteEx ig) {
+ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+
+ assertTrue(pages < 19100);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ef8e4799/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
new file mode 100644
index 0000000..d9e3f34
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+
+import javax.cache.expiry.CreatedExpiryPolicy;
+import javax.cache.expiry.Duration;
+import javax.cache.expiry.ExpiryPolicy;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+/**
+ *
+ */
+public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakAbstractTest {
+
+ private static final ExpiryPolicy EXPIRY = new CreatedExpiryPolicy(new Duration(MILLISECONDS, 10L));
+
+ @Override
+ protected int duration() {
+ return 300;
+ }
+
+ @Override
+ protected int gridCount() {
+ return 1;
+ }
+
+ @Override
+ protected void configure(IgniteConfiguration cfg) {
+ cfg.setMetricsLogFrequency(5000);
+ }
+
+ @Override
+ protected void configure(MemoryConfiguration mCfg) {
+ mCfg.setPageCacheSize(1024 * 1024);
+ }
+
+ @Override
+ protected boolean indexingEnabled() {
+ return false;
+ }
+
+ protected void operation(IgniteEx ig) {
+ IgniteCache<Object, Object> cache = ig.cache("non-primitive").withExpiryPolicy(EXPIRY);
+ Random rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < 1000; i++) {
+ DbKey key = new DbKey(rnd.nextInt(200_000));
+
+ DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
+
+ switch (rnd.nextInt(3)) {
+ case 0:
+ cache.getAndPut(key, v0);
+ case 1:
+ cache.get(key);
+ break;
+ case 2:
+ cache.getAndRemove(key);
+ }
+ }
+ }
+
+ @Override
+ protected void check(IgniteEx ig) {
+ long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+
+ assertTrue(pages < 10000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ef8e4799/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
index c7a07e3..228a262 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
@@ -17,175 +17,39 @@
package org.apache.ignite.internal.processors.database;
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ThreadLocalRandom;
-import javax.cache.Cache;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CachePeekMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cache.affinity.AffinityFunction;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.cache.query.SqlQuery;
-import org.apache.ignite.cache.query.annotations.QuerySqlField;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
import org.apache.ignite.internal.util.GridRandom;
import org.apache.ignite.internal.util.typedef.PA;
import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.junit.Assert;
+import javax.cache.Cache;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+
/**
*
*/
-public abstract class IgniteDbPutGetAbstractTest extends GridCommonAbstractTest {
- /** */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /**
- * @return Node count.
- */
- protected abstract int gridCount();
-
- /**
- * @return {@code True} if indexing is enabled.
- */
- protected abstract boolean indexingEnabled();
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- MemoryConfiguration dbCfg = new MemoryConfiguration();
-
- if (isLargePage()) {
- dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
-
- dbCfg.setPageSize(16 * 1024);
-
- dbCfg.setPageCacheSize(200 * 1024 * 1024);
- }
- else {
- dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
-
- dbCfg.setPageSize(1024);
-
- dbCfg.setPageCacheSize(200 * 1024 * 1024);
- }
-
- cfg.setMemoryConfiguration(dbCfg);
-
- CacheConfiguration ccfg = new CacheConfiguration();
-
- if (indexingEnabled())
- ccfg.setIndexedTypes(Integer.class, DbValue.class);
-
- ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
- ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- CacheConfiguration ccfg2 = new CacheConfiguration("non-primitive");
-
- if (indexingEnabled())
- ccfg2.setIndexedTypes(DbKey.class, DbValue.class);
-
- ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg2.setRebalanceMode(CacheRebalanceMode.SYNC);
- ccfg2.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- CacheConfiguration ccfg3 = new CacheConfiguration("large");
-
- if (indexingEnabled())
- ccfg3.setIndexedTypes(Integer.class, LargeDbValue.class);
-
- ccfg3.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg3.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg3.setRebalanceMode(CacheRebalanceMode.SYNC);
- ccfg3.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- CacheConfiguration ccfg4 = new CacheConfiguration("tiny");
-
- ccfg4.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg4.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg4.setRebalanceMode(CacheRebalanceMode.SYNC);
- ccfg4.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- final AffinityFunction aff = new RendezvousAffinityFunction(1, null);
-
- ccfg4.setAffinity(aff);
-
- cfg.setCacheConfiguration(ccfg, ccfg2, ccfg3, ccfg4);
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(IP_FINDER);
-
- cfg.setDiscoverySpi(discoSpi);
- cfg.setMarshaller(null);
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
-
- long seed = 1464583813940L; // System.currentTimeMillis();
-
- info("Seed: " + seed + "L");
-
- BPlusTree.rnd = new Random(seed);
-
- startGrids(gridCount());
-
- awaitPartitionMapExchange();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- BPlusTree.rnd = null;
-
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /**
- * @return {@code True} if use large page.
- */
- protected boolean isLargePage() {
- return false;
- };
-
+public abstract class IgniteDbPutGetAbstractTest extends IgniteDbAbstractTest {
/**
*
*/
@@ -1349,191 +1213,4 @@ public abstract class IgniteDbPutGetAbstractTest extends GridCommonAbstractTest
assertNull(internalCache.peekEx(key));
}
-
- /**
- *
- */
- private static class DbKey implements Serializable {
- /** */
- private int val;
-
- /**
- * @param val Value.
- */
- private DbKey(int val) {
- this.val = val;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || !(o instanceof DbKey))
- return false;
-
- DbKey key = (DbKey)o;
-
- return val == key.val;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return val;
- }
- }
-
- /**
- *
- */
- private static class LargeDbKey implements Serializable {
- /** */
- private int val;
-
- /** */
- private byte[] data;
-
- /**
- * @param val Value.
- * @param size Key payload size.
- */
- private LargeDbKey(int val, int size) {
- this.val = val;
-
- data = new byte[size];
-
- Arrays.fill(data, (byte)val);
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || !(o instanceof LargeDbKey))
- return false;
-
- LargeDbKey key = (LargeDbKey)o;
-
- return val == key.val && Arrays.equals(data, key.data);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return val + Arrays.hashCode(data);
- }
- }
-
- /**
- *
- */
- private static class DbValue implements Serializable {
- /** */
- @QuerySqlField(index = true)
- private int iVal;
-
- /** */
- @QuerySqlField(index = true)
- private String sVal;
-
- /** */
- @QuerySqlField
- private long lVal;
-
- /**
- * @param iVal Integer value.
- * @param sVal String value.
- * @param lVal Long value.
- */
- public DbValue(int iVal, String sVal, long lVal) {
- this.iVal = iVal;
- this.sVal = sVal;
- this.lVal = lVal;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- DbValue dbVal = (DbValue)o;
-
- return iVal == dbVal.iVal && lVal == dbVal.lVal &&
- !(sVal != null ? !sVal.equals(dbVal.sVal) : dbVal.sVal != null);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = iVal;
-
- res = 31 * res + (sVal != null ? sVal.hashCode() : 0);
- res = 31 * res + (int)(lVal ^ (lVal >>> 32));
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(DbValue.class, this);
- }
- }
-
- /**
- *
- */
- private static class LargeDbValue {
- /** */
- @QuerySqlField(index = true)
- private String str1;
-
- /** */
- @QuerySqlField(index = true)
- private String str2;
-
- /** */
- private int[] arr;
-
- /**
- * @param str1 String 1.
- * @param str2 String 2.
- * @param arr Big array.
- */
- public LargeDbValue(final String str1, final String str2, final int[] arr) {
- this.str1 = str1;
- this.str2 = str2;
- this.arr = arr;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(final Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- final LargeDbValue that = (LargeDbValue) o;
-
- if (str1 != null ? !str1.equals(that.str1) : that.str1 != null) return false;
- if (str2 != null ? !str2.equals(that.str2) : that.str2 != null) return false;
-
- return Arrays.equals(arr, that.arr);
-
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = str1 != null ? str1.hashCode() : 0;
-
- res = 31 * res + (str2 != null ? str2.hashCode() : 0);
- res = 31 * res + Arrays.hashCode(arr);
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(LargeDbValue.class, this);
- }
- }
}
[03/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4376f82d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4376f82d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4376f82d
Branch: refs/heads/ignite-4712
Commit: 4376f82d13ef4d08c4eb8d9b6e3f19cba9d9ce1c
Parents: be241d9
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Mon Feb 20 16:40:25 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Tue Feb 21 12:44:29 2017 +0300
----------------------------------------------------------------------
.../cache/database/freelist/PagesList.java | 30 ++++++++++++++++++--
1 file changed, 28 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4376f82d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
index 1f537b6..5ae3549 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
@@ -189,6 +189,7 @@ public abstract class PagesList extends DataStructure {
for (Map.Entry<Integer, GridLongList> e : bucketsData.entrySet()) {
int bucket = e.getKey();
+ long bucketSize = 0;
Stripe[] old = getBucket(bucket);
assert old == null;
@@ -197,11 +198,36 @@ public abstract class PagesList extends DataStructure {
Stripe[] tails = new Stripe[upd.length];
- for (int i = 0; i < upd.length; i++)
- tails[i] = new Stripe(upd[i]);
+ for (int i = 0; i < upd.length; i++) {
+ long tailId = upd[i];
+
+ try(Page tail = page(tailId)) {
+ long tailAddr = readLock(tail);
+
+ assert tailAddr != 0L;
+
+ try {
+ PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(tailAddr);
+
+ int count = io.getCount(tailAddr);
+
+ Stripe stripe = new Stripe(tailId);
+ stripe.empty = count == 0;
+
+ tails[i] = stripe;
+
+ bucketSize += count;
+ }
+ finally {
+ readUnlock(tail, tailAddr);
+ }
+ }
+ }
boolean ok = casBucket(bucket, null, tails);
assert ok;
+
+ bucketsSize[bucket].set(bucketSize);
}
}
}
[31/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b3d78b5c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b3d78b5c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b3d78b5c
Branch: refs/heads/ignite-4712
Commit: b3d78b5c4ac8142d443b15af96c64c9fa20f5b43
Parents: 5e11f5d
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Mon Feb 20 16:19:00 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:54 2017 +0300
----------------------------------------------------------------------
.../cache/database/freelist/PagesList.java | 44 +++++++++++++-------
1 file changed, 28 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b3d78b5c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
index c03f432..1f537b6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
@@ -22,6 +22,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.internal.pagemem.Page;
@@ -76,7 +77,7 @@ public abstract class PagesList extends DataStructure {
Math.min(8, Runtime.getRuntime().availableProcessors() * 2));
/** */
- protected final LongAdder8[] bucketsSize;
+ protected final AtomicLong[] bucketsSize;
/** Page ID to store list metadata. */
private final long metaPageId;
@@ -114,7 +115,7 @@ public abstract class PagesList extends DataStructure {
return TRUE;
}
- };
+ }
/**
* @param cacheId Cache ID.
@@ -139,10 +140,10 @@ public abstract class PagesList extends DataStructure {
this.buckets = buckets;
this.metaPageId = metaPageId;
- bucketsSize = new LongAdder8[buckets];
+ bucketsSize = new AtomicLong[buckets];
for (int i = 0; i < buckets; i++)
- bucketsSize[i] = new LongAdder8();
+ bucketsSize[i] = new AtomicLong();
}
/**
@@ -421,6 +422,7 @@ public abstract class PagesList extends DataStructure {
else {
// It is safe to assign new tail since we do it only when write lock on tail is held.
tails[idx].tailId = newTailId;
+ tails[idx].empty = false;
return true;
}
@@ -597,7 +599,7 @@ public abstract class PagesList extends DataStructure {
if (idx == -1)
handlePageFull(pageId, page, pageAddr, io, dataPage, dataPageAddr, bucket);
else {
- bucketsSize[bucket].increment();
+ bucketsSize[bucket].incrementAndGet();
if (isWalDeltaRecordNeeded(wal, page))
wal.log(new PagesListAddPageRecord(cacheId, pageId, dataPageId));
@@ -691,7 +693,7 @@ public abstract class PagesList extends DataStructure {
assert idx != -1;
- bucketsSize[bucket].increment();
+ bucketsSize[bucket].incrementAndGet();
dataIO.setFreeListPageId(dataPageAddr, nextId);
@@ -781,7 +783,7 @@ public abstract class PagesList extends DataStructure {
}
}
else {
- bucketsSize[bucket].increment();
+ bucketsSize[bucket].incrementAndGet();
// TODO: use single WAL record for bag?
if (isWalDeltaRecordNeeded(wal, page))
@@ -810,7 +812,7 @@ public abstract class PagesList extends DataStructure {
private Stripe getPageForTake(int bucket) {
Stripe[] tails = getBucket(bucket);
- if (tails == null)
+ if (tails == null || bucketsSize[bucket].get() == 0)
return null;
int len = tails.length;
@@ -893,6 +895,16 @@ public abstract class PagesList extends DataStructure {
continue;
}
+ if(!isReuseBucket(bucket) && stripe.empty) {
+ // Another thread took the last page
+ writeUnlock(tail, tailPageAddr, false);
+
+ if(bucketsSize[bucket].get() > 0)
+ continue;
+ else
+ return 0L;
+ }
+
assert PageIO.getPageId(tailPageAddr) == tailId : "tailId = " + tailId + ", tailPageId = " + PageIO.getPageId(tailPageAddr);
assert PageIO.getType(tailPageAddr) == PageIO.T_PAGE_LIST_NODE;
@@ -909,7 +921,7 @@ public abstract class PagesList extends DataStructure {
long pageId = io.takeAnyPage(tailPageAddr);
if (pageId != 0L) {
- bucketsSize[bucket].decrement();
+ bucketsSize[bucket].decrementAndGet();
if (isWalDeltaRecordNeeded(wal, tail))
wal.log(new PagesListRemovePageRecord(cacheId, tailId, pageId));
@@ -918,9 +930,13 @@ public abstract class PagesList extends DataStructure {
ret = pageId;
+ boolean empty = io.isEmpty(tailPageAddr);
+
+ stripe.empty = empty;
+
// If we got an empty page in non-reuse bucket, move it back to reuse list
// to prevent empty page leak to data pages.
- if (io.isEmpty(tailPageAddr) && !isReuseBucket(bucket)) {
+ if (empty && !isReuseBucket(bucket)) {
long prevId = io.getPreviousId(tailPageAddr);
if (prevId != 0L) {
@@ -969,12 +985,8 @@ public abstract class PagesList extends DataStructure {
ret = tailId;
}
- else {
+ else
stripe.empty = true;
-
- if (bucketsSize[bucket].sum() > 0)
- continue;
- }
}
// If we do not have a previous page (we are at head), then we still can return
@@ -1034,7 +1046,7 @@ public abstract class PagesList extends DataStructure {
if (!rmvd)
return false;
- bucketsSize[bucket].decrement();
+ bucketsSize[bucket].decrementAndGet();
if (isWalDeltaRecordNeeded(wal, page))
wal.log(new PagesListRemovePageRecord(cacheId, pageId, dataPageId));
[24/43] ignite git commit: IGNITE-4694 Add tests to check there are
no memory leaks in PageMemory (pending)
Posted by sb...@apache.org.
IGNITE-4694 Add tests to check there are no memory leaks in PageMemory (pending)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6743d5c6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6743d5c6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6743d5c6
Branch: refs/heads/ignite-4712
Commit: 6743d5c6b7f7a8b71b9560acf06a86d6cb031523
Parents: 25c1927
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Thu Feb 16 13:41:51 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:54 2017 +0300
----------------------------------------------------------------------
.../IgniteDbMemoryLeakAbstractTest.java | 118 +++++++++++++++----
.../database/IgniteDbMemoryLeakIndexedTest.java | 29 -----
.../IgniteDbMemoryLeakLargeObjectsTest.java | 16 +--
.../IgniteDbMemoryLeakLargePagesTest.java | 13 +-
.../database/IgniteDbMemoryLeakTest.java | 14 ++-
.../IgniteDbMemoryLeakWithExpirationTest.java | 7 +-
.../database/IgniteDbMemoryLeakIndexedTest.java | 42 +++++++
7 files changed, 165 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index bca3af0..819405e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -30,13 +30,37 @@ import org.jetbrains.annotations.NotNull;
* Base class for memory leaks tests.
*/
public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest {
-
+ /** */
+ @SuppressWarnings("WeakerAccess") protected static final int CONCURRENCY_LEVEL = 8;
/** */
private volatile Exception ex = null;
/** */
+ private long warmUpEndTime;
+
+ /** */
+ private long endTime;
+
+ /** */
+ private long loadedPages = 0;
+
+ /** */
+ private long delta = 0;
+
+ /** */
+ private long probeCnt = 0;
+
+ /** */
private static final ThreadLocal<Random> THREAD_LOCAL_RANDOM = new ThreadLocal<>();
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ long startTime = System.nanoTime();
+ warmUpEndTime = startTime + TimeUnit.SECONDS.toNanos(warmUp());
+ endTime = warmUpEndTime + TimeUnit.SECONDS.toNanos(duration());
+ }
+
/** {@inheritDoc} */
@Override protected void configure(IgniteConfiguration cfg) {
cfg.setMetricsLogFrequency(5000);
@@ -44,10 +68,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/** {@inheritDoc} */
@Override protected void configure(MemoryConfiguration mCfg) {
- int concLvl = Runtime.getRuntime().availableProcessors();
-
- mCfg.setConcurrencyLevel(concLvl);
- mCfg.setPageCacheSize(1024 * 1024 * concLvl); //minimal possible value
+ mCfg.setConcurrencyLevel(CONCURRENCY_LEVEL);
}
/**
@@ -57,6 +78,13 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
return 300;
}
+ /**
+ * @return Warm up duration.
+ */
+ @SuppressWarnings("WeakerAccess") protected int warmUp() {
+ return 300;
+ }
+
/** {@inheritDoc} */
@Override protected int gridCount() {
return 1;
@@ -69,14 +97,14 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/** {@inheritDoc} */
@Override protected long getTestTimeout() {
- return (duration() + 1) * 1000;
+ return (warmUp() + duration() + 1) * 1000; // One extra second to stop all threads
}
/**
* @param ig Ignite instance.
* @return IgniteCache.
*/
- protected abstract IgniteCache<Object,Object> cache(IgniteEx ig);
+ protected abstract IgniteCache<Object, Object> cache(IgniteEx ig);
/**
* @return Cache key to perform an operation.
@@ -84,8 +112,8 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
protected abstract Object key();
/**
- * @return Cache value to perform an operation.
* @param key Cache key to perform an operation.
+ * @return Cache value to perform an operation.
*/
protected abstract Object value(Object key);
@@ -99,8 +127,11 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
switch (getRandom().nextInt(3)) {
case 0:
cache.getAndPut(key, value);
+
+ break;
case 1:
cache.get(key);
+
break;
case 2:
cache.getAndRemove(key);
@@ -113,7 +144,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
@NotNull protected static Random getRandom() {
Random rnd = THREAD_LOCAL_RANDOM.get();
- if(rnd == null){
+ if (rnd == null) {
rnd = new GridRandom();
THREAD_LOCAL_RANDOM.set(rnd);
}
@@ -125,49 +156,96 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
* @throws Exception If failed.
*/
public void testMemoryLeak() throws Exception {
- final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration());
final IgniteEx ignite = grid(0);
final IgniteCache<Object, Object> cache = cache(ignite);
Runnable target = new Runnable() {
@Override public void run() {
- while (ex == null && System.nanoTime() < end) {
+ while (ex == null && System.nanoTime() < endTime) {
try {
operation(cache);
- check(ignite);
}
catch (Exception e) {
ex = e;
-
break;
}
}
}
};
- Thread[] threads = new Thread[Runtime.getRuntime().availableProcessors()];
+ Thread[] threads = new Thread[CONCURRENCY_LEVEL];
+
+ info("Warming up is started.");
for (int i = 0; i < threads.length; i++) {
threads[i] = new Thread(target);
threads[i].start();
}
- for (Thread thread : threads) {
- thread.join();
+ Thread.sleep(TimeUnit.NANOSECONDS.toMillis(warmUpEndTime - System.nanoTime()));
+
+ info("Warming up is ended.");
+
+ while (System.nanoTime() < endTime) {
+ try {
+ check(ignite);
+ }
+ catch (Exception e) {
+ ex = e;
+
+ break;
+ }
+
+ Thread.sleep(TimeUnit.SECONDS.toMillis(5));
}
- if(ex != null){
+ for (Thread thread : threads)
+ thread.join();
+
+ if (ex != null)
throw ex;
- }
}
/**
- * Callback to check the current state
+ * Callback to check the current state.
*
- * @param ig Ignite instance
+ * @param ig Ignite instance.
* @throws Exception If failed.
*/
protected void check(IgniteEx ig) throws Exception {
+ long pagesActual = ig.context().cache().context().database().pageMemory().loadedPages();
+ long pagesMax = pagesMax();
+
+ assertTrue(
+ "Maximal allowed pages number is exceeded. [allowed=" + pagesMax + "; actual= " + pagesActual + "]",
+ pagesActual <= pagesMax);
+
+ if (loadedPages > 0) {
+ delta += pagesActual - loadedPages;
+ int allowedDelta = pagesDelta();
+
+ if(probeCnt++ > 12) { // we need some statistic first. Minimal statistic is taken for a minute.
+ long actualDelta = delta / probeCnt;
+
+ assertTrue(
+ "Average growth pages in the number is more than expected. [allowed=" + allowedDelta + "; actual=" + actualDelta + "]",
+ actualDelta <= allowedDelta);
+ }
+ }
+
+ loadedPages = pagesActual;
+ }
+
+ /**
+ * @return Maximal allowed pages number.
+ */
+ protected abstract long pagesMax();
+
+ /**
+ * @return Expected average number of pages, on which their total number can grow per 5 seconds.
+ */
+ @SuppressWarnings("WeakerAccess") protected int pagesDelta() {
+ return 5;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
deleted file mode 100644
index acc6c2f..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.database;
-
-/**
- *
- */
-public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakTest {
-
- /** {@inheritDoc} */
- @Override protected boolean indexingEnabled() {
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
index 8943743..c1b1c9b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
@@ -36,23 +36,25 @@ public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstra
/** {@inheritDoc} */
@Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
- return ig.cache("non-primitive");
+ return ig.cache("large");
}
/** {@inheritDoc} */
@Override protected Object key() {
- return new DbKey(getRandom().nextInt(200_000));
+ return new LargeDbKey(getRandom().nextInt(10_000), 1024);
}
/** {@inheritDoc} */
@Override protected Object value(Object key) {
- return new DbValue(((DbKey)key).val, "test-value-" + getRandom().nextInt(200), getRandom().nextInt(500));
+ return new LargeDbValue("test-value-1-" + getRandom().nextInt(200), "test-value-2-" + getRandom().nextInt(200), ARRAY);
}
- /** {@inheritDoc} */
- @Override protected void check(IgniteEx ig) {
- long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+ @Override protected int warmUp() {
+ return 600;
+ }
- assertTrue(pages < 20000);
+ /** {@inheritDoc} */
+ @Override protected long pagesMax() {
+ return 2000000;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
index 8e4d0b4..9dd9fdc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
@@ -18,19 +18,16 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.internal.IgniteEx;
/**
*
*/
public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakTest {
-
/** {@inheritDoc} */
@Override protected void configure(MemoryConfiguration mCfg) {
- int concLvl = Runtime.getRuntime().availableProcessors();
- mCfg.setConcurrencyLevel(concLvl);
- mCfg.setPageCacheSize(1024 * 1024 * concLvl * 16);
+ super.configure(mCfg);
+ mCfg.setPageCacheSize(67108864); // The space for 4000 large pages
}
/** {@inheritDoc} */
@@ -39,9 +36,7 @@ public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakTest {
}
/** {@inheritDoc} */
- @Override protected void check(IgniteEx ig) {
- long pages = ig.context().cache().context().database().pageMemory().loadedPages();
-
- assertTrue(pages < 4000);
+ @Override protected long pagesMax() {
+ return 4000;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
index 81d831b..94e419d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
/**
@@ -25,6 +26,13 @@ import org.apache.ignite.internal.IgniteEx;
*/
public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
/** {@inheritDoc} */
+ @Override protected void configure(MemoryConfiguration mCfg) {
+ super.configure(mCfg);
+
+ mCfg.setPageCacheSize(20971520); // The space for 20000 pages
+ }
+
+ /** {@inheritDoc} */
@Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
return ig.cache("non-primitive");
}
@@ -40,9 +48,7 @@ public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
}
/** {@inheritDoc} */
- @Override protected void check(IgniteEx ig) {
- long pages = ig.context().cache().context().database().pageMemory().loadedPages();
-
- assertTrue(pages < 20000);
+ @Override protected long pagesMax() {
+ return 20_000;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
index a31ffb4..c1e279b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
@@ -39,10 +39,7 @@ public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakTest
return ig.cache("non-primitive").withExpiryPolicy(EXPIRY);
}
- /** {@inheritDoc} */
- @Override protected void check(IgniteEx ig) {
- long pages = ig.context().cache().context().database().pageMemory().loadedPages();
-
- assertTrue(pages < 7000);
+ @Override protected long pagesMax() {
+ return 7000;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
new file mode 100644
index 0000000..c917bc1
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.configuration.MemoryConfiguration;
+
+/**
+ *
+ */
+public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakTest {
+ /** {@inheritDoc} */
+ @Override protected void configure(MemoryConfiguration mCfg) {
+ super.configure(mCfg);
+
+ mCfg.setPageCacheSize(25165824); // The space for 24000 pages
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean indexingEnabled() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long pagesMax() {
+ return 24_000;
+ }
+}
[39/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4e7912ab
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4e7912ab
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4e7912ab
Branch: refs/heads/ignite-4712
Commit: 4e7912ab62c3e0a69b505a3216987692b3cd8c01
Parents: 7d6bf32
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Wed Feb 22 11:28:14 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:55 2017 +0300
----------------------------------------------------------------------
.../cache/database/freelist/PagesList.java | 38 ++++++++------------
1 file changed, 14 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4e7912ab/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
index 5fdd191..b2c6e9c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
@@ -200,23 +200,17 @@ public abstract class PagesList extends DataStructure {
for (int i = 0; i < upd.length; i++) {
long tailId = upd[i];
- // TODO: just relase readlock.
- List<Page> locked = new ArrayList<>(2);
- List<Long> lockedAddrs = new ArrayList<>(2);
+ long pageId = tailId;
+ int count = 0;
- try {
- long pageId = tailId;
- int count = 0;
-
- while (pageId != 0L) {
- try (Page page = page(pageId)) {
- long pageAddr = readLock(page);
+ while (pageId != 0L) {
- assert pageAddr != 0L;
+ try (Page page = page(pageId)) {
+ long pageAddr = readLock(page);
- locked.add(page);
- lockedAddrs.add(pageAddr);
+ assert pageAddr != 0L;
+ try {
PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(pageAddr);
count += io.getCount(pageAddr);
@@ -226,19 +220,15 @@ public abstract class PagesList extends DataStructure {
if (isReuseBucket(bucket) && pageId != 0L)
count++;
}
- }
-
- Stripe stripe = new Stripe(tailId, count == 0);
-
- tails[i] = stripe;
-
- bucketSize += count;
- }
- finally {
- for (int j = 0; j < locked.size(); j++) {
- readUnlock(locked.get(j), lockedAddrs.get(j));
+ finally {
+ readUnlock(page, pageAddr);
+ }
}
}
+
+ Stripe stripe = new Stripe(tailId, count == 0);
+ tails[i] = stripe;
+ bucketSize += count;
}
boolean ok = casBucket(bucket, null, tails);
[15/43] ignite git commit: ignite-4712
Posted by sb...@apache.org.
ignite-4712
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/852183e1
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/852183e1
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/852183e1
Branch: refs/heads/ignite-4712
Commit: 852183e111667e6740b46f400e78fe9598ffac1a
Parents: fd19998
Author: sboikov <sb...@gridgain.com>
Authored: Wed Feb 22 11:15:04 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Wed Feb 22 11:15:04 2017 +0300
----------------------------------------------------------------------
.../cache/database/freelist/PagesList.java | 59 ++++++++++++++++----
1 file changed, 47 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/852183e1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
index a5fb617..5fdd191 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
@@ -52,7 +52,6 @@ import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.jetbrains.annotations.Nullable;
-import org.jsr166.LongAdder8;
import static java.lang.Boolean.FALSE;
import static java.lang.Boolean.TRUE;
@@ -201,6 +200,7 @@ public abstract class PagesList extends DataStructure {
for (int i = 0; i < upd.length; i++) {
long tailId = upd[i];
+ // TODO: just relase readlock.
List<Page> locked = new ArrayList<>(2);
List<Long> lockedAddrs = new ArrayList<>(2);
@@ -222,14 +222,13 @@ public abstract class PagesList extends DataStructure {
count += io.getCount(pageAddr);
pageId = io.getPreviousId(pageAddr);
- // In reuse bucket the page itself can be used as a free page
- if(isReuseBucket(bucket) && pageId != 0L)
+ // In reuse bucket the page itself can be used as a free page.
+ if (isReuseBucket(bucket) && pageId != 0L)
count++;
}
}
- Stripe stripe = new Stripe(tailId);
- stripe.empty = count == 0;
+ Stripe stripe = new Stripe(tailId, count == 0);
tails[i] = stripe;
@@ -405,8 +404,7 @@ public abstract class PagesList extends DataStructure {
initPage(pageMem, page, this, PagesListNodeIO.VERSIONS.latest(), wal);
}
- Stripe stripe = new Stripe(pageId);
- stripe.empty = true;
+ Stripe stripe = new Stripe(pageId, true);
for (;;) {
Stripe[] old = getBucket(bucket);
@@ -518,11 +516,43 @@ public abstract class PagesList extends DataStructure {
}
/**
+ * !!! For tests only, does not provide any correctness guarantees for concurrent access.
+ *
* @param bucket Bucket index.
* @return Number of pages stored in this list.
+ * @throws IgniteCheckedException If failed.
*/
- protected final long storedPagesCount(int bucket) {
- return bucketsSize[bucket].get();
+ protected final long storedPagesCount(int bucket) throws IgniteCheckedException {
+ long res = 0;
+
+ Stripe[] tails = getBucket(bucket);
+
+ if (tails != null) {
+ for (Stripe tail : tails) {
+ long pageId = tail.tailId;
+
+ try (Page page = page(pageId)) {
+ long pageAddr = readLock(page); // No correctness guaranties.
+
+ try {
+ PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(pageAddr);
+
+ int cnt = io.getCount(pageAddr);
+
+ assert cnt >= 0;
+
+ res += cnt;
+ }
+ finally {
+ readUnlock(page, pageAddr);
+ }
+ }
+ }
+ }
+
+ assert res == bucketsSize[bucket].get();
+
+ return res;
}
/**
@@ -672,6 +702,7 @@ public abstract class PagesList extends DataStructure {
dataPageId,
pageId, 0L));
+ // In reuse bucket the page itself can be used as a free page.
bucketsSize[bucket].incrementAndGet();
updateTail(bucket, pageId, dataPageId);
@@ -791,7 +822,8 @@ public abstract class PagesList extends DataStructure {
0L
));
- if(isReuseBucket(bucket))
+ // In reuse bucket the page itself can be used as a free page.
+ if (isReuseBucket(bucket))
bucketsSize[bucket].incrementAndGet();
// Switch to this new page, which is now a part of our list
@@ -938,9 +970,10 @@ public abstract class PagesList extends DataStructure {
try {
PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(tailPageAddr);
- if (io.getNextId(tailPageAddr) != 0)
+ if (io.getNextId(tailPageAddr) != 0) {
// It is not a tail anymore, retry.
continue;
+ }
long pageId = io.takeAnyPage(tailPageAddr);
@@ -1356,9 +1389,11 @@ public abstract class PagesList extends DataStructure {
/**
* @param tailId Tail ID.
+ * @param empty Empty flag.
*/
- Stripe(long tailId) {
+ Stripe(long tailId, boolean empty) {
this.tailId = tailId;
+ this.empty = empty;
}
}
}
[16/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1a2bbef1
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1a2bbef1
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1a2bbef1
Branch: refs/heads/ignite-4712
Commit: 1a2bbef110e1d6f924576c91c4e73a58174cb581
Parents: 852183e
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Wed Feb 22 11:28:14 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Feb 22 11:28:14 2017 +0300
----------------------------------------------------------------------
.../cache/database/freelist/PagesList.java | 38 ++++++++------------
1 file changed, 14 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a2bbef1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
index 5fdd191..b2c6e9c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
@@ -200,23 +200,17 @@ public abstract class PagesList extends DataStructure {
for (int i = 0; i < upd.length; i++) {
long tailId = upd[i];
- // TODO: just relase readlock.
- List<Page> locked = new ArrayList<>(2);
- List<Long> lockedAddrs = new ArrayList<>(2);
+ long pageId = tailId;
+ int count = 0;
- try {
- long pageId = tailId;
- int count = 0;
-
- while (pageId != 0L) {
- try (Page page = page(pageId)) {
- long pageAddr = readLock(page);
+ while (pageId != 0L) {
- assert pageAddr != 0L;
+ try (Page page = page(pageId)) {
+ long pageAddr = readLock(page);
- locked.add(page);
- lockedAddrs.add(pageAddr);
+ assert pageAddr != 0L;
+ try {
PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(pageAddr);
count += io.getCount(pageAddr);
@@ -226,19 +220,15 @@ public abstract class PagesList extends DataStructure {
if (isReuseBucket(bucket) && pageId != 0L)
count++;
}
- }
-
- Stripe stripe = new Stripe(tailId, count == 0);
-
- tails[i] = stripe;
-
- bucketSize += count;
- }
- finally {
- for (int j = 0; j < locked.size(); j++) {
- readUnlock(locked.get(j), lockedAddrs.get(j));
+ finally {
+ readUnlock(page, pageAddr);
+ }
}
}
+
+ Stripe stripe = new Stripe(tailId, count == 0);
+ tails[i] = stripe;
+ bucketSize += count;
}
boolean ok = casBucket(bucket, null, tails);
[37/43] ignite git commit: ignite-4712 review
Posted by sb...@apache.org.
ignite-4712 review
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b5389f7a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b5389f7a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b5389f7a
Branch: refs/heads/ignite-4712
Commit: b5389f7a6976ed04717d773ea7577e3f64061e65
Parents: b9fcc14
Author: sboikov <sb...@gridgain.com>
Authored: Tue Feb 21 17:32:29 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:55 2017 +0300
----------------------------------------------------------------------
.../cache/database/freelist/PagesList.java | 19 ++++--
.../database/IgniteDbAbstractTest.java | 68 +++++++++-----------
.../IgniteDbMemoryLeakAbstractTest.java | 36 +++++++----
.../IgniteDbMemoryLeakWithExpirationTest.java | 5 +-
.../IgniteDbMemoryLeakSqlQueryTest.java | 17 +++--
5 files changed, 80 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b5389f7a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
index 5ae3549..7c88041 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
@@ -201,7 +201,8 @@ public abstract class PagesList extends DataStructure {
for (int i = 0; i < upd.length; i++) {
long tailId = upd[i];
- try(Page tail = page(tailId)) {
+ // TODO: need get full bucket size.
+ try (Page tail = page(tailId)) {
long tailAddr = readLock(tail);
assert tailAddr != 0L;
@@ -847,10 +848,11 @@ public abstract class PagesList extends DataStructure {
while (true) {
Stripe stripe = tails[cur];
- if(!stripe.empty)
+
+ if (!stripe.empty)
return stripe;
- if((cur = (cur + 1) % len) == init)
+ if ((cur = (cur + 1) % len) == init)
return null;
}
}
@@ -921,12 +923,16 @@ public abstract class PagesList extends DataStructure {
continue;
}
- if(!isReuseBucket(bucket) && stripe.empty) {
- // Another thread took the last page
+ // TODO: condition !isReuseBucket(bucket) is not correct.
+ if (!isReuseBucket(bucket) && stripe.empty) {
+ // Another thread took the last page.
writeUnlock(tail, tailPageAddr, false);
- if(bucketsSize[bucket].get() > 0)
+ if (bucketsSize[bucket].get() > 0) {
+ lockAttempt = 0;
+
continue;
+ }
else
return 0L;
}
@@ -958,6 +964,7 @@ public abstract class PagesList extends DataStructure {
boolean empty = io.isEmpty(tailPageAddr);
+ // TODO: add comment, it seems flag is not set to correct value for reuse bucket.
stripe.empty = empty;
// If we got an empty page in non-reuse bucket, move it back to reuse list
http://git-wip-us.apache.org/repos/asf/ignite/blob/b5389f7a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
index 5f4d5e0..46efab2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
@@ -17,10 +17,8 @@
package org.apache.ignite.internal.processors.database;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.AffinityFunction;
+import java.io.Serializable;
+import java.util.Arrays;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
import org.apache.ignite.configuration.CacheConfiguration;
@@ -34,9 +32,10 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.Random;
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheRebalanceMode.SYNC;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
/**
*
@@ -56,7 +55,8 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
protected abstract boolean indexingEnabled();
/** {@inheritDoc} */
- @SuppressWarnings("unchecked") @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ @SuppressWarnings("unchecked")
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(gridName);
MemoryConfiguration dbCfg = new MemoryConfiguration();
@@ -80,9 +80,9 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
if (indexingEnabled())
ccfg.setIndexedTypes(Integer.class, DbValue.class);
- ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg.setAtomicityMode(TRANSACTIONAL);
+ ccfg.setWriteSynchronizationMode(FULL_SYNC);
+ ccfg.setRebalanceMode(SYNC);
ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
CacheConfiguration ccfg2 = new CacheConfiguration("non-primitive");
@@ -90,9 +90,9 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
if (indexingEnabled())
ccfg2.setIndexedTypes(DbKey.class, DbValue.class);
- ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg2.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg2.setAtomicityMode(TRANSACTIONAL);
+ ccfg2.setWriteSynchronizationMode(FULL_SYNC);
+ ccfg2.setRebalanceMode(SYNC);
ccfg2.setAffinity(new RendezvousAffinityFunction(false, 32));
CacheConfiguration ccfg3 = new CacheConfiguration("large");
@@ -100,32 +100,28 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
if (indexingEnabled())
ccfg3.setIndexedTypes(Integer.class, LargeDbValue.class);
- ccfg3.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg3.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg3.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg3.setAtomicityMode(TRANSACTIONAL);
+ ccfg3.setWriteSynchronizationMode(FULL_SYNC);
+ ccfg3.setRebalanceMode(SYNC);
ccfg3.setAffinity(new RendezvousAffinityFunction(false, 32));
CacheConfiguration ccfg4 = new CacheConfiguration("tiny");
- ccfg4.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg4.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg4.setRebalanceMode(CacheRebalanceMode.SYNC);
- ccfg4.setAffinity(new RendezvousAffinityFunction(false, 32));
+ ccfg4.setAtomicityMode(TRANSACTIONAL);
+ ccfg4.setWriteSynchronizationMode(FULL_SYNC);
+ ccfg4.setRebalanceMode(SYNC);
+ ccfg4.setAffinity(new RendezvousAffinityFunction(1, null));
CacheConfiguration ccfg5 = new CacheConfiguration("atomic");
if (indexingEnabled())
ccfg5.setIndexedTypes(DbKey.class, DbValue.class);
- ccfg5.setAtomicityMode(CacheAtomicityMode.ATOMIC);
- ccfg5.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg5.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg5.setAtomicityMode(ATOMIC);
+ ccfg5.setWriteSynchronizationMode(FULL_SYNC);
+ ccfg5.setRebalanceMode(SYNC);
ccfg5.setAffinity(new RendezvousAffinityFunction(false, 32));
- final AffinityFunction aff = new RendezvousAffinityFunction(1, null);
-
- ccfg4.setAffinity(aff);
-
cfg.setCacheConfiguration(ccfg, ccfg2, ccfg3, ccfg4, ccfg5);
TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
@@ -144,25 +140,25 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
* @param cfg IgniteConfiguration.
*/
protected void configure(IgniteConfiguration cfg){
- //NOP
+ // No-op.
}
/**
* @param mCfg MemoryConfiguration.
*/
protected void configure(MemoryConfiguration mCfg){
- //NOP
+ // No-op.
}
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- long seed = 1464583813940L; // System.currentTimeMillis();
-
- info("Seed: " + seed + "L");
-
- BPlusTree.rnd = new Random(seed);
+// long seed = System.currentTimeMillis();
+//
+// info("Seed: " + seed + "L");
+//
+// BPlusTree.rnd = new Random(seed);
startGrids(gridCount());
@@ -275,8 +271,6 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
@QuerySqlField
long lVal;
-
-
/**
* @param iVal Integer value.
* @param sVal String value.
http://git-wip-us.apache.org/repos/asf/ignite/blob/b5389f7a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index 93306d9..aa32495 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -30,13 +30,13 @@ import org.apache.ignite.internal.processors.cache.database.DataStructure;
*/
public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest {
/** */
- @SuppressWarnings("WeakerAccess") protected static final int CONCURRENCY_LEVEL = 8;
+ private static final int CONCURRENCY_LEVEL = 8;
/** */
private static final int MIN_PAGE_CACHE_SIZE = 1048576 * CONCURRENCY_LEVEL;
/** */
- private volatile Exception ex = null;
+ private volatile Exception ex;
/** */
private long warmUpEndTime;
@@ -45,21 +45,24 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
private long endTime;
/** */
- private long loadedPages = 0;
+ private long loadedPages;
/** */
- private long delta = 0;
+ private long delta;
/** */
- private long probeCnt = 0;
+ private long probeCnt;
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
super.beforeTest();
+
DataStructure.rnd = null;
long startTime = System.nanoTime();
+
warmUpEndTime = startTime + TimeUnit.SECONDS.toNanos(warmUp());
+
endTime = warmUpEndTime + TimeUnit.SECONDS.toNanos(duration());
}
@@ -73,6 +76,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
mCfg.setConcurrencyLevel(CONCURRENCY_LEVEL);
long size = 1024 * pagesMax() * (isLargePage() ? 16 : 1);
+
mCfg.setPageCacheSize(Math.max(size, MIN_PAGE_CACHE_SIZE));
}
@@ -86,7 +90,8 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/**
* @return Warm up duration in seconds.
*/
- @SuppressWarnings("WeakerAccess") protected int warmUp() {
+ @SuppressWarnings("WeakerAccess")
+ protected int warmUp() {
return 300;
}
@@ -102,7 +107,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/** {@inheritDoc} */
@Override protected long getTestTimeout() {
- return (warmUp() + duration() + 1) * 2000; // Two extra seconds to stop all threads.
+ return (warmUp() + duration() + 10) * 1000; // Extra seconds to stop all threads.
}
/**
@@ -127,17 +132,19 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
*/
protected void operation(IgniteCache<Object, Object> cache) {
Object key = key();
- Object value = value(key);
+ Object val = value(key);
switch (nextInt(3)) {
case 0:
- cache.getAndPut(key, value);
+ cache.getAndPut(key, val);
break;
+
case 1:
cache.get(key);
break;
+
case 2:
cache.getAndRemove(key);
}
@@ -173,6 +180,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
}
catch (Exception e) {
ex = e;
+
break;
}
}
@@ -223,18 +231,19 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
long pagesMax = pagesMax();
assertTrue(
- "Maximal allowed pages number is exceeded. [allowed=" + pagesMax + "; actual= " + pagesActual + "]",
+ "Maximal allowed pages number is exceeded [allowed=" + pagesMax + ", actual= " + pagesActual + "]",
pagesActual <= pagesMax);
if (loadedPages > 0) {
delta += pagesActual - loadedPages;
+
int allowedDelta = pagesDelta();
- if(probeCnt++ > 12) { // we need some statistic first. Minimal statistic is taken for a minute.
+ if (probeCnt++ > 12) { // We need some statistic first. Minimal statistic is taken for a minute.
long actualDelta = delta / probeCnt;
assertTrue(
- "Average growth pages in the number is more than expected. [allowed=" + allowedDelta + "; actual=" + actualDelta + "]",
+ "Average growth pages in the number is more than expected [allowed=" + allowedDelta + ", actual=" + actualDelta + "]",
actualDelta <= allowedDelta);
}
}
@@ -250,7 +259,8 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/**
* @return Expected average number of pages, on which their total number can grow per 5 seconds.
*/
- @SuppressWarnings("WeakerAccess") protected int pagesDelta() {
+ @SuppressWarnings("WeakerAccess")
+ protected int pagesDelta() {
return 3;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b5389f7a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
index 19502b0..6e0abaf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
@@ -17,12 +17,11 @@
package org.apache.ignite.internal.processors.database;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.internal.IgniteEx;
-
import javax.cache.expiry.CreatedExpiryPolicy;
import javax.cache.expiry.Duration;
import javax.cache.expiry.ExpiryPolicy;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.internal.IgniteEx;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
http://git-wip-us.apache.org/repos/asf/ignite/blob/b5389f7a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakSqlQueryTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakSqlQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakSqlQueryTest.java
index b65cac0..57f9fb5 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakSqlQueryTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakSqlQueryTest.java
@@ -38,21 +38,24 @@ public class IgniteDbMemoryLeakSqlQueryTest extends IgniteDbMemoryLeakTest {
/** {@inheritDoc} */
@Override protected void operation(IgniteCache<Object, Object> cache) {
Object key = key();
- Object value = value(key);
+ Object val = value(key);
switch (nextInt(4)) {
case 0:
- cache.getAndPut(key, value);
+ cache.getAndPut(key, val);
break;
+
case 1:
cache.get(key);
break;
+
case 2:
cache.getAndRemove(key);
break;
+
case 3:
cache.query(sqlQuery(cache)).getAll();
}
@@ -63,9 +66,11 @@ public class IgniteDbMemoryLeakSqlQueryTest extends IgniteDbMemoryLeakTest {
* @return SqlFieldsQuery.
*/
@NotNull private SqlFieldsQuery sqlQuery(IgniteCache<Object, Object> cache) {
- String query = String.format("select _key from \"%s\".DbValue where iVal=?", cache.getName());
- SqlFieldsQuery sqlQuery = new SqlFieldsQuery(query);
- sqlQuery.setArgs(nextInt(200_000));
- return sqlQuery;
+ String qry = String.format("select _key from \"%s\".DbValue where iVal=?", cache.getName());
+
+ SqlFieldsQuery sqlQry = new SqlFieldsQuery(qry);
+ sqlQry.setArgs(nextInt(200_000));
+
+ return sqlQry;
}
}
[36/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/cae3a224
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/cae3a224
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/cae3a224
Branch: refs/heads/ignite-4712
Commit: cae3a224021eea2000ab72b4771b22b2c265fda4
Parents: b5389f7
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Wed Feb 22 10:59:06 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:55 2017 +0300
----------------------------------------------------------------------
.../cache/database/freelist/PagesList.java | 178 +++++++++----------
1 file changed, 85 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/cae3a224/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
index 7c88041..a5fb617 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
@@ -201,26 +201,43 @@ public abstract class PagesList extends DataStructure {
for (int i = 0; i < upd.length; i++) {
long tailId = upd[i];
- // TODO: need get full bucket size.
- try (Page tail = page(tailId)) {
- long tailAddr = readLock(tail);
+ List<Page> locked = new ArrayList<>(2);
+ List<Long> lockedAddrs = new ArrayList<>(2);
- assert tailAddr != 0L;
+ try {
+ long pageId = tailId;
+ int count = 0;
+
+ while (pageId != 0L) {
+ try (Page page = page(pageId)) {
+ long pageAddr = readLock(page);
- try {
- PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(tailAddr);
+ assert pageAddr != 0L;
- int count = io.getCount(tailAddr);
+ locked.add(page);
+ lockedAddrs.add(pageAddr);
- Stripe stripe = new Stripe(tailId);
- stripe.empty = count == 0;
+ PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(pageAddr);
- tails[i] = stripe;
+ count += io.getCount(pageAddr);
+ pageId = io.getPreviousId(pageAddr);
- bucketSize += count;
+ // In reuse bucket the page itself can be used as a free page
+ if(isReuseBucket(bucket) && pageId != 0L)
+ count++;
+ }
}
- finally {
- readUnlock(tail, tailAddr);
+
+ Stripe stripe = new Stripe(tailId);
+ stripe.empty = count == 0;
+
+ tails[i] = stripe;
+
+ bucketSize += count;
+ }
+ finally {
+ for (int j = 0; j < locked.size(); j++) {
+ readUnlock(locked.get(j), lockedAddrs.get(j));
}
}
}
@@ -389,6 +406,7 @@ public abstract class PagesList extends DataStructure {
}
Stripe stripe = new Stripe(pageId);
+ stripe.empty = true;
for (;;) {
Stripe[] old = getBucket(bucket);
@@ -449,7 +467,6 @@ public abstract class PagesList extends DataStructure {
else {
// It is safe to assign new tail since we do it only when write lock on tail is held.
tails[idx].tailId = newTailId;
- tails[idx].empty = false;
return true;
}
@@ -501,41 +518,11 @@ public abstract class PagesList extends DataStructure {
}
/**
- * !!! For tests only, does not provide any correctness guarantees for concurrent access.
- *
* @param bucket Bucket index.
* @return Number of pages stored in this list.
- * @throws IgniteCheckedException If failed.
*/
- protected final long storedPagesCount(int bucket) throws IgniteCheckedException {
- long res = 0;
-
- Stripe[] tails = getBucket(bucket);
-
- if (tails != null) {
- for (Stripe tail : tails) {
- long pageId = tail.tailId;
-
- try (Page page = page(pageId)) {
- long pageAddr = readLock(page); // No correctness guaranties.
-
- try {
- PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(pageAddr);
-
- int cnt = io.getCount(pageAddr);
-
- assert cnt >= 0;
-
- res += cnt;
- }
- finally {
- readUnlock(page, pageAddr);
- }
- }
- }
- }
-
- return res;
+ protected final long storedPagesCount(int bucket) {
+ return bucketsSize[bucket].get();
}
/**
@@ -685,6 +672,8 @@ public abstract class PagesList extends DataStructure {
dataPageId,
pageId, 0L));
+ bucketsSize[bucket].incrementAndGet();
+
updateTail(bucket, pageId, dataPageId);
}
else {
@@ -720,13 +709,13 @@ public abstract class PagesList extends DataStructure {
assert idx != -1;
- bucketsSize[bucket].incrementAndGet();
-
dataIO.setFreeListPageId(dataPageAddr, nextId);
if (isWalDeltaRecordNeeded(wal, dataPage))
wal.log(new DataPageSetFreeListPageRecord(cacheId, dataPageId, nextId));
+ bucketsSize[bucket].incrementAndGet();
+
updateTail(bucket, pageId, nextId);
}
finally {
@@ -802,6 +791,9 @@ public abstract class PagesList extends DataStructure {
0L
));
+ if(isReuseBucket(bucket))
+ bucketsSize[bucket].incrementAndGet();
+
// Switch to this new page, which is now a part of our list
// to add the rest of the bag to the new page.
prevPageAddr = nextPageAddr;
@@ -810,11 +802,11 @@ public abstract class PagesList extends DataStructure {
}
}
else {
- bucketsSize[bucket].incrementAndGet();
-
// TODO: use single WAL record for bag?
if (isWalDeltaRecordNeeded(wal, page))
wal.log(new PagesListAddPageRecord(cacheId, prevId, nextId));
+
+ bucketsSize[bucket].incrementAndGet();
}
}
}
@@ -923,13 +915,12 @@ public abstract class PagesList extends DataStructure {
continue;
}
- // TODO: condition !isReuseBucket(bucket) is not correct.
- if (!isReuseBucket(bucket) && stripe.empty) {
+ if (stripe.empty) {
// Another thread took the last page.
writeUnlock(tail, tailPageAddr, false);
if (bucketsSize[bucket].get() > 0) {
- lockAttempt = 0;
+ lockAttempt--; // Ignore current attempt.
continue;
}
@@ -941,13 +932,14 @@ public abstract class PagesList extends DataStructure {
assert PageIO.getType(tailPageAddr) == PageIO.T_PAGE_LIST_NODE;
boolean dirty = false;
- long ret = 0L;
+ long ret;
long recycleId = 0L;
try {
PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(tailPageAddr);
if (io.getNextId(tailPageAddr) != 0)
+ // It is not a tail anymore, retry.
continue;
long pageId = io.takeAnyPage(tailPageAddr);
@@ -962,64 +954,64 @@ public abstract class PagesList extends DataStructure {
ret = pageId;
- boolean empty = io.isEmpty(tailPageAddr);
-
- // TODO: add comment, it seems flag is not set to correct value for reuse bucket.
- stripe.empty = empty;
-
- // If we got an empty page in non-reuse bucket, move it back to reuse list
- // to prevent empty page leak to data pages.
- if (empty && !isReuseBucket(bucket)) {
+ if (io.isEmpty(tailPageAddr)) {
long prevId = io.getPreviousId(tailPageAddr);
- if (prevId != 0L) {
- try (Page prev = page(prevId)) {
- // Lock pages from next to previous.
- Boolean ok = writePage(pageMem, prev, this, cutTail, null, bucket, FALSE);
+ // If we got an empty page in non-reuse bucket, move it back to reuse list
+ // to prevent empty page leak to data pages.
+ if (!isReuseBucket(bucket)) {
+ if (prevId != 0L) {
+ try (Page prev = page(prevId)) {
+ // Lock pages from next to previous.
+ Boolean ok = writePage(pageMem, prev, this, cutTail, null, bucket, FALSE);
- assert ok == TRUE : ok;
- }
+ assert ok == TRUE : ok;
+ }
- recycleId = recyclePage(tailId, tail, tailPageAddr);
+ recycleId = recyclePage(tailId, tail, tailPageAddr);
+ }
+ else
+ stripe.empty = true;
}
+ else
+ stripe.empty = prevId == 0L;
}
}
else {
- // The tail page is empty. We can unlink and return it if we have a previous page.
+ // The tail page is empty, but stripe is not. It might
+ // happen only if we are in reuse bucket and it has
+ // a previous page, so, the current page can be collected
+ assert isReuseBucket(bucket);
+
long prevId = io.getPreviousId(tailPageAddr);
- if (prevId != 0L) {
- // This can only happen if we are in the reuse bucket.
- assert isReuseBucket(bucket);
+ assert prevId != 0L;
- try (Page prev = page(prevId)) {
- // Lock pages from next to previous.
- Boolean ok = writePage(pageMem, prev, this, cutTail, null, bucket, FALSE);
+ try (Page prev = page(prevId)) {
+ // Lock pages from next to previous.
+ Boolean ok = writePage(pageMem, prev, this, cutTail, null, bucket, FALSE);
- assert ok == TRUE : ok;
- }
+ assert ok == TRUE : ok;
+ }
- if (initIoVers != null) {
- tailId = PageIdUtils.changeType(tailId, FLAG_DATA);
+ if (initIoVers != null) {
+ tailId = PageIdUtils.changeType(tailId, FLAG_DATA);
- PageIO initIo = initIoVers.latest();
+ PageIO initIo = initIoVers.latest();
- initIo.initNewPage(tailPageAddr, tailId, pageSize());
+ initIo.initNewPage(tailPageAddr, tailId, pageSize());
- if (isWalDeltaRecordNeeded(wal, tail)) {
- wal.log(new InitNewPageRecord(cacheId, tail.id(), initIo.getType(),
- initIo.getVersion(), tailId));
- }
+ if (isWalDeltaRecordNeeded(wal, tail)) {
+ wal.log(new InitNewPageRecord(cacheId, tail.id(), initIo.getType(),
+ initIo.getVersion(), tailId));
}
- else
- tailId = recyclePage(tailId, tail, tailPageAddr);
-
- dirty = true;
-
- ret = tailId;
}
else
- stripe.empty = true;
+ tailId = recyclePage(tailId, tail, tailPageAddr);
+
+ dirty = true;
+
+ ret = tailId;
}
// If we do not have a previous page (we are at head), then we still can return
[35/43] ignite git commit: IGNITE-4694 Add tests to check there are
no memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4694 Add tests to check there are no memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/27178f13
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/27178f13
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/27178f13
Branch: refs/heads/ignite-4712
Commit: 27178f1394efed5a0e72e4a66d19bd2c9d1bb5c1
Parents: af05cd8
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Wed Feb 22 13:18:18 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:55 2017 +0300
----------------------------------------------------------------------
.../IgniteDbMemoryLeakAbstractTest.java | 22 +++++++++-----------
1 file changed, 10 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/27178f13/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index aa32495..04ac5ea 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -75,7 +75,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
@Override protected void configure(MemoryConfiguration mCfg) {
mCfg.setConcurrencyLevel(CONCURRENCY_LEVEL);
- long size = 1024 * pagesMax() * (isLargePage() ? 16 : 1);
+ long size = (1024 * (isLargePage() ? 16 : 1) + 24) * pagesMax();
mCfg.setPageCacheSize(Math.max(size, MIN_PAGE_CACHE_SIZE));
}
@@ -92,7 +92,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
*/
@SuppressWarnings("WeakerAccess")
protected int warmUp() {
- return 300;
+ return 450;
}
/** {@inheritDoc} */
@@ -172,6 +172,8 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
final IgniteEx ignite = grid(0);
final IgniteCache<Object, Object> cache = cache(ignite);
+
+
Runnable target = new Runnable() {
@Override public void run() {
while (ex == null && System.nanoTime() < endTime) {
@@ -196,11 +198,15 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
threads[i].start();
}
- Thread.sleep(TimeUnit.NANOSECONDS.toMillis(warmUpEndTime - System.nanoTime()));
+ while (ex == null && System.nanoTime() < warmUpEndTime)
+ Thread.sleep(100);
+
+ if (ex != null)
+ throw ex;
info("Warming up is ended.");
- while (System.nanoTime() < endTime) {
+ while (ex == null && System.nanoTime() < endTime) {
try {
check(ignite);
}
@@ -213,9 +219,6 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
Thread.sleep(TimeUnit.SECONDS.toMillis(5));
}
- for (Thread thread : threads)
- thread.join();
-
if (ex != null)
throw ex;
}
@@ -228,11 +231,6 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
*/
protected void check(IgniteEx ig) throws Exception {
long pagesActual = ig.context().cache().context().database().pageMemory().loadedPages();
- long pagesMax = pagesMax();
-
- assertTrue(
- "Maximal allowed pages number is exceeded [allowed=" + pagesMax + ", actual= " + pagesActual + "]",
- pagesActual <= pagesMax);
if (loadedPages > 0) {
delta += pagesActual - loadedPages;
[11/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4ae7d717
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4ae7d717
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4ae7d717
Branch: refs/heads/ignite-4712
Commit: 4ae7d7177b1e55c31579c4691514655eeadb833f
Parents: bfb0021
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Mon Feb 20 18:31:15 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Tue Feb 21 12:44:30 2017 +0300
----------------------------------------------------------------------
.../processors/database/IgniteDbMemoryLeakAbstractTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4ae7d717/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index bef1320..ccb1fa0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -103,7 +103,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/** {@inheritDoc} */
@Override protected long getTestTimeout() {
- return (warmUp() + duration() + 1) * 1000; // One extra second to stop all threads
+ return (warmUp() + duration() + 1) * 2000; // Two extra seconds to stop all threads.
}
/**
[13/43] ignite git commit: ignite-4712 review
Posted by sb...@apache.org.
ignite-4712 review
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d376ea98
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d376ea98
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d376ea98
Branch: refs/heads/ignite-4712
Commit: d376ea9862bbe3be7d9f4a53f2237e55fb92364f
Parents: 5e4e991
Author: sboikov <sb...@gridgain.com>
Authored: Tue Feb 21 17:32:29 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Tue Feb 21 17:32:29 2017 +0300
----------------------------------------------------------------------
.../cache/database/freelist/PagesList.java | 19 ++++--
.../database/IgniteDbAbstractTest.java | 68 +++++++++-----------
.../IgniteDbMemoryLeakAbstractTest.java | 36 +++++++----
.../IgniteDbMemoryLeakWithExpirationTest.java | 5 +-
.../IgniteDbMemoryLeakSqlQueryTest.java | 17 +++--
5 files changed, 80 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d376ea98/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
index 5ae3549..7c88041 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
@@ -201,7 +201,8 @@ public abstract class PagesList extends DataStructure {
for (int i = 0; i < upd.length; i++) {
long tailId = upd[i];
- try(Page tail = page(tailId)) {
+ // TODO: need get full bucket size.
+ try (Page tail = page(tailId)) {
long tailAddr = readLock(tail);
assert tailAddr != 0L;
@@ -847,10 +848,11 @@ public abstract class PagesList extends DataStructure {
while (true) {
Stripe stripe = tails[cur];
- if(!stripe.empty)
+
+ if (!stripe.empty)
return stripe;
- if((cur = (cur + 1) % len) == init)
+ if ((cur = (cur + 1) % len) == init)
return null;
}
}
@@ -921,12 +923,16 @@ public abstract class PagesList extends DataStructure {
continue;
}
- if(!isReuseBucket(bucket) && stripe.empty) {
- // Another thread took the last page
+ // TODO: condition !isReuseBucket(bucket) is not correct.
+ if (!isReuseBucket(bucket) && stripe.empty) {
+ // Another thread took the last page.
writeUnlock(tail, tailPageAddr, false);
- if(bucketsSize[bucket].get() > 0)
+ if (bucketsSize[bucket].get() > 0) {
+ lockAttempt = 0;
+
continue;
+ }
else
return 0L;
}
@@ -958,6 +964,7 @@ public abstract class PagesList extends DataStructure {
boolean empty = io.isEmpty(tailPageAddr);
+ // TODO: add comment, it seems flag is not set to correct value for reuse bucket.
stripe.empty = empty;
// If we got an empty page in non-reuse bucket, move it back to reuse list
http://git-wip-us.apache.org/repos/asf/ignite/blob/d376ea98/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
index 5f4d5e0..46efab2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
@@ -17,10 +17,8 @@
package org.apache.ignite.internal.processors.database;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.AffinityFunction;
+import java.io.Serializable;
+import java.util.Arrays;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
import org.apache.ignite.configuration.CacheConfiguration;
@@ -34,9 +32,10 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.Random;
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheRebalanceMode.SYNC;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
/**
*
@@ -56,7 +55,8 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
protected abstract boolean indexingEnabled();
/** {@inheritDoc} */
- @SuppressWarnings("unchecked") @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ @SuppressWarnings("unchecked")
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(gridName);
MemoryConfiguration dbCfg = new MemoryConfiguration();
@@ -80,9 +80,9 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
if (indexingEnabled())
ccfg.setIndexedTypes(Integer.class, DbValue.class);
- ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg.setAtomicityMode(TRANSACTIONAL);
+ ccfg.setWriteSynchronizationMode(FULL_SYNC);
+ ccfg.setRebalanceMode(SYNC);
ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
CacheConfiguration ccfg2 = new CacheConfiguration("non-primitive");
@@ -90,9 +90,9 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
if (indexingEnabled())
ccfg2.setIndexedTypes(DbKey.class, DbValue.class);
- ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg2.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg2.setAtomicityMode(TRANSACTIONAL);
+ ccfg2.setWriteSynchronizationMode(FULL_SYNC);
+ ccfg2.setRebalanceMode(SYNC);
ccfg2.setAffinity(new RendezvousAffinityFunction(false, 32));
CacheConfiguration ccfg3 = new CacheConfiguration("large");
@@ -100,32 +100,28 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
if (indexingEnabled())
ccfg3.setIndexedTypes(Integer.class, LargeDbValue.class);
- ccfg3.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg3.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg3.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg3.setAtomicityMode(TRANSACTIONAL);
+ ccfg3.setWriteSynchronizationMode(FULL_SYNC);
+ ccfg3.setRebalanceMode(SYNC);
ccfg3.setAffinity(new RendezvousAffinityFunction(false, 32));
CacheConfiguration ccfg4 = new CacheConfiguration("tiny");
- ccfg4.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg4.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg4.setRebalanceMode(CacheRebalanceMode.SYNC);
- ccfg4.setAffinity(new RendezvousAffinityFunction(false, 32));
+ ccfg4.setAtomicityMode(TRANSACTIONAL);
+ ccfg4.setWriteSynchronizationMode(FULL_SYNC);
+ ccfg4.setRebalanceMode(SYNC);
+ ccfg4.setAffinity(new RendezvousAffinityFunction(1, null));
CacheConfiguration ccfg5 = new CacheConfiguration("atomic");
if (indexingEnabled())
ccfg5.setIndexedTypes(DbKey.class, DbValue.class);
- ccfg5.setAtomicityMode(CacheAtomicityMode.ATOMIC);
- ccfg5.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg5.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg5.setAtomicityMode(ATOMIC);
+ ccfg5.setWriteSynchronizationMode(FULL_SYNC);
+ ccfg5.setRebalanceMode(SYNC);
ccfg5.setAffinity(new RendezvousAffinityFunction(false, 32));
- final AffinityFunction aff = new RendezvousAffinityFunction(1, null);
-
- ccfg4.setAffinity(aff);
-
cfg.setCacheConfiguration(ccfg, ccfg2, ccfg3, ccfg4, ccfg5);
TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
@@ -144,25 +140,25 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
* @param cfg IgniteConfiguration.
*/
protected void configure(IgniteConfiguration cfg){
- //NOP
+ // No-op.
}
/**
* @param mCfg MemoryConfiguration.
*/
protected void configure(MemoryConfiguration mCfg){
- //NOP
+ // No-op.
}
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- long seed = 1464583813940L; // System.currentTimeMillis();
-
- info("Seed: " + seed + "L");
-
- BPlusTree.rnd = new Random(seed);
+// long seed = System.currentTimeMillis();
+//
+// info("Seed: " + seed + "L");
+//
+// BPlusTree.rnd = new Random(seed);
startGrids(gridCount());
@@ -275,8 +271,6 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
@QuerySqlField
long lVal;
-
-
/**
* @param iVal Integer value.
* @param sVal String value.
http://git-wip-us.apache.org/repos/asf/ignite/blob/d376ea98/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index 93306d9..aa32495 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -30,13 +30,13 @@ import org.apache.ignite.internal.processors.cache.database.DataStructure;
*/
public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest {
/** */
- @SuppressWarnings("WeakerAccess") protected static final int CONCURRENCY_LEVEL = 8;
+ private static final int CONCURRENCY_LEVEL = 8;
/** */
private static final int MIN_PAGE_CACHE_SIZE = 1048576 * CONCURRENCY_LEVEL;
/** */
- private volatile Exception ex = null;
+ private volatile Exception ex;
/** */
private long warmUpEndTime;
@@ -45,21 +45,24 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
private long endTime;
/** */
- private long loadedPages = 0;
+ private long loadedPages;
/** */
- private long delta = 0;
+ private long delta;
/** */
- private long probeCnt = 0;
+ private long probeCnt;
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
super.beforeTest();
+
DataStructure.rnd = null;
long startTime = System.nanoTime();
+
warmUpEndTime = startTime + TimeUnit.SECONDS.toNanos(warmUp());
+
endTime = warmUpEndTime + TimeUnit.SECONDS.toNanos(duration());
}
@@ -73,6 +76,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
mCfg.setConcurrencyLevel(CONCURRENCY_LEVEL);
long size = 1024 * pagesMax() * (isLargePage() ? 16 : 1);
+
mCfg.setPageCacheSize(Math.max(size, MIN_PAGE_CACHE_SIZE));
}
@@ -86,7 +90,8 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/**
* @return Warm up duration in seconds.
*/
- @SuppressWarnings("WeakerAccess") protected int warmUp() {
+ @SuppressWarnings("WeakerAccess")
+ protected int warmUp() {
return 300;
}
@@ -102,7 +107,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/** {@inheritDoc} */
@Override protected long getTestTimeout() {
- return (warmUp() + duration() + 1) * 2000; // Two extra seconds to stop all threads.
+ return (warmUp() + duration() + 10) * 1000; // Extra seconds to stop all threads.
}
/**
@@ -127,17 +132,19 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
*/
protected void operation(IgniteCache<Object, Object> cache) {
Object key = key();
- Object value = value(key);
+ Object val = value(key);
switch (nextInt(3)) {
case 0:
- cache.getAndPut(key, value);
+ cache.getAndPut(key, val);
break;
+
case 1:
cache.get(key);
break;
+
case 2:
cache.getAndRemove(key);
}
@@ -173,6 +180,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
}
catch (Exception e) {
ex = e;
+
break;
}
}
@@ -223,18 +231,19 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
long pagesMax = pagesMax();
assertTrue(
- "Maximal allowed pages number is exceeded. [allowed=" + pagesMax + "; actual= " + pagesActual + "]",
+ "Maximal allowed pages number is exceeded [allowed=" + pagesMax + ", actual= " + pagesActual + "]",
pagesActual <= pagesMax);
if (loadedPages > 0) {
delta += pagesActual - loadedPages;
+
int allowedDelta = pagesDelta();
- if(probeCnt++ > 12) { // we need some statistic first. Minimal statistic is taken for a minute.
+ if (probeCnt++ > 12) { // We need some statistic first. Minimal statistic is taken for a minute.
long actualDelta = delta / probeCnt;
assertTrue(
- "Average growth pages in the number is more than expected. [allowed=" + allowedDelta + "; actual=" + actualDelta + "]",
+ "Average growth pages in the number is more than expected [allowed=" + allowedDelta + ", actual=" + actualDelta + "]",
actualDelta <= allowedDelta);
}
}
@@ -250,7 +259,8 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/**
* @return Expected average number of pages, on which their total number can grow per 5 seconds.
*/
- @SuppressWarnings("WeakerAccess") protected int pagesDelta() {
+ @SuppressWarnings("WeakerAccess")
+ protected int pagesDelta() {
return 3;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/d376ea98/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
index 19502b0..6e0abaf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
@@ -17,12 +17,11 @@
package org.apache.ignite.internal.processors.database;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.internal.IgniteEx;
-
import javax.cache.expiry.CreatedExpiryPolicy;
import javax.cache.expiry.Duration;
import javax.cache.expiry.ExpiryPolicy;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.internal.IgniteEx;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
http://git-wip-us.apache.org/repos/asf/ignite/blob/d376ea98/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakSqlQueryTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakSqlQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakSqlQueryTest.java
index b65cac0..57f9fb5 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakSqlQueryTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakSqlQueryTest.java
@@ -38,21 +38,24 @@ public class IgniteDbMemoryLeakSqlQueryTest extends IgniteDbMemoryLeakTest {
/** {@inheritDoc} */
@Override protected void operation(IgniteCache<Object, Object> cache) {
Object key = key();
- Object value = value(key);
+ Object val = value(key);
switch (nextInt(4)) {
case 0:
- cache.getAndPut(key, value);
+ cache.getAndPut(key, val);
break;
+
case 1:
cache.get(key);
break;
+
case 2:
cache.getAndRemove(key);
break;
+
case 3:
cache.query(sqlQuery(cache)).getAll();
}
@@ -63,9 +66,11 @@ public class IgniteDbMemoryLeakSqlQueryTest extends IgniteDbMemoryLeakTest {
* @return SqlFieldsQuery.
*/
@NotNull private SqlFieldsQuery sqlQuery(IgniteCache<Object, Object> cache) {
- String query = String.format("select _key from \"%s\".DbValue where iVal=?", cache.getName());
- SqlFieldsQuery sqlQuery = new SqlFieldsQuery(query);
- sqlQuery.setArgs(nextInt(200_000));
- return sqlQuery;
+ String qry = String.format("select _key from \"%s\".DbValue where iVal=?", cache.getName());
+
+ SqlFieldsQuery sqlQry = new SqlFieldsQuery(qry);
+ sqlQry.setArgs(nextInt(200_000));
+
+ return sqlQry;
}
}
[05/43] ignite git commit: IGNITE-4694 Add tests to check there are
no memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4694 Add tests to check there are no memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e70d990f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e70d990f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e70d990f
Branch: refs/heads/ignite-4712
Commit: e70d990f14288cfc8fe211fa25631016d5708144
Parents: 8e12097
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Wed Feb 15 18:04:38 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Tue Feb 21 12:44:29 2017 +0300
----------------------------------------------------------------------
.../cache/IgniteCacheOffheapManagerImpl.java | 2 +-
.../database/IgniteDbAbstractTest.java | 6 +
.../IgniteDbMemoryLeakAbstractTest.java | 172 ++++++++++++++-----
.../database/IgniteDbMemoryLeakIndexedTest.java | 65 +------
.../IgniteDbMemoryLeakLargeObjectsTest.java | 64 ++-----
.../IgniteDbMemoryLeakLargePagesTest.java | 67 ++------
.../database/IgniteDbMemoryLeakTest.java | 63 ++-----
.../IgniteDbMemoryLeakWithExpirationTest.java | 58 +------
8 files changed, 182 insertions(+), 315 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e70d990f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index 5df99b6..9becc99 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -897,7 +897,7 @@ public class IgniteCacheOffheapManagerImpl extends GridCacheManagerAdapter imple
*/
private boolean canUpdateOldRow(@Nullable CacheDataRow oldRow, DataRow dataRow)
throws IgniteCheckedException {
- if (oldRow == null || indexingEnabled)
+ if (oldRow == null || indexingEnabled || oldRow.expireTime() != dataRow.expireTime())
return false;
CacheObjectContext coCtx = cctx.cacheObjectContext();
http://git-wip-us.apache.org/repos/asf/ignite/blob/e70d990f/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
index 3bc7004..9297cec 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
@@ -130,10 +130,16 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
return cfg;
}
+ /**
+ * @param cfg IgniteConfiguration.
+ */
protected void configure(IgniteConfiguration cfg){
//NOP
}
+ /**
+ * @param mCfg MemoryConfiguration.
+ */
protected void configure(MemoryConfiguration mCfg){
//NOP
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e70d990f/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index fc0e715..bca3af0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -17,69 +17,157 @@
package org.apache.ignite.internal.processors.database;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCompute;
-import org.apache.ignite.compute.ComputeTaskFuture;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.lang.IgniteRunnable;
-import org.apache.ignite.resources.IgniteInstanceResource;
-
+import java.util.Random;
import java.util.concurrent.TimeUnit;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.GridRandom;
+import org.jetbrains.annotations.NotNull;
/**
- * TODO: fix javadoc warnings, code style.
+ * Base class for memory leaks tests.
*/
public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest {
- // TODO: take duration from system property.
- /** Test duration in seconds*/
- protected abstract int duration();
-
- @Override
- protected long getTestTimeout() {
- return duration() * 1200;
- }
/** */
- protected abstract void operation(IgniteEx ig);
+ private volatile Exception ex = null;
/** */
- public void testMemoryLeak() throws Exception {
- // TODO: take PageMemory max size is the same as we configured.
+ private static final ThreadLocal<Random> THREAD_LOCAL_RANDOM = new ThreadLocal<>();
- final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration());
+ /** {@inheritDoc} */
+ @Override protected void configure(IgniteConfiguration cfg) {
+ cfg.setMetricsLogFrequency(5000);
+ }
- // TODO: use threads instead of compute or make sure there are enough threads in pool.
- int tasksCnt = Runtime.getRuntime().availableProcessors() * 4;
+ /** {@inheritDoc} */
+ @Override protected void configure(MemoryConfiguration mCfg) {
+ int concLvl = Runtime.getRuntime().availableProcessors();
- IgniteCompute compute = grid(0).compute().withAsync();
+ mCfg.setConcurrencyLevel(concLvl);
+ mCfg.setPageCacheSize(1024 * 1024 * concLvl); //minimal possible value
+ }
- ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCnt];
+ /**
+ * @return Test duration in seconds.
+ */
+ protected int duration() {
+ return 300;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int gridCount() {
+ return 1;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean indexingEnabled() {
+ return false;
+ }
- for (int i = 0; i < tasksCnt; i++) {
- compute.run(new IgniteRunnable() {
- @IgniteInstanceResource
- private Ignite ig;
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return (duration() + 1) * 1000;
+ }
- @Override
- public void run() {
- int i = 0;
- while (System.nanoTime() < end) {
- operation((IgniteEx) ig);
+ /**
+ * @param ig Ignite instance.
+ * @return IgniteCache.
+ */
+ protected abstract IgniteCache<Object,Object> cache(IgniteEx ig);
+
+ /**
+ * @return Cache key to perform an operation.
+ */
+ protected abstract Object key();
+
+ /**
+ * @return Cache value to perform an operation.
+ * @param key Cache key to perform an operation.
+ */
+ protected abstract Object value(Object key);
+
+ /**
+ * @param cache IgniteCache.
+ */
+ protected void operation(IgniteCache<Object, Object> cache) {
+ Object key = key();
+ Object value = value(key);
+
+ switch (getRandom().nextInt(3)) {
+ case 0:
+ cache.getAndPut(key, value);
+ case 1:
+ cache.get(key);
+ break;
+ case 2:
+ cache.getAndRemove(key);
+ }
+ }
- if(i++ == 100) {
- check((IgniteEx) ig);
- i = 0;
- }
+ /**
+ * @return Random.
+ */
+ @NotNull protected static Random getRandom() {
+ Random rnd = THREAD_LOCAL_RANDOM.get();
+
+ if(rnd == null){
+ rnd = new GridRandom();
+ THREAD_LOCAL_RANDOM.set(rnd);
+ }
+
+ return rnd;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testMemoryLeak() throws Exception {
+ final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration());
+
+ final IgniteEx ignite = grid(0);
+ final IgniteCache<Object, Object> cache = cache(ignite);
+
+ Runnable target = new Runnable() {
+ @Override public void run() {
+ while (ex == null && System.nanoTime() < end) {
+ try {
+ operation(cache);
+ check(ignite);
+ }
+ catch (Exception e) {
+ ex = e;
+
+ break;
}
}
- });
+ }
+ };
+
+ Thread[] threads = new Thread[Runtime.getRuntime().availableProcessors()];
- futs[i] = compute.future();
+ for (int i = 0; i < threads.length; i++) {
+ threads[i] = new Thread(target);
+ threads[i].start();
}
- for (ComputeTaskFuture fut : futs)
- fut.get();
+ for (Thread thread : threads) {
+ thread.join();
+ }
+
+ if(ex != null){
+ throw ex;
+ }
}
- protected void check(IgniteEx ig) {}
+ /**
+ * Callback to check the current state
+ *
+ * @param ig Ignite instance
+ * @throws Exception If failed.
+ */
+ protected void check(IgniteEx ig) throws Exception {
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e70d990f/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
index db77131..acc6c2f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
@@ -17,72 +17,13 @@
package org.apache.ignite.internal.processors.database;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
-
/**
*
*/
-public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakAbstractTest {
-
- @Override
- protected int duration() {
- return 300;
- }
-
- @Override
- protected int gridCount() {
- return 1;
- }
-
- @Override
- protected void configure(IgniteConfiguration cfg) {
- cfg.setMetricsLogFrequency(5000);
- }
-
- @Override
- protected void configure(MemoryConfiguration mCfg) {
- mCfg.setPageCacheSize(1024 * 1024);
- }
+public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakTest {
- @Override
- // TODO: move test to module ignite-indexing.
- protected boolean indexingEnabled() {
+ /** {@inheritDoc} */
+ @Override protected boolean indexingEnabled() {
return true;
}
-
- protected void operation(IgniteEx ig){
- IgniteCache<Object, Object> cache = ig.cache("non-primitive");
- Random rnd = ThreadLocalRandom.current();
-
- for (int i = 0; i < 1000; i++) {
- DbKey key = new DbKey(rnd.nextInt(200_000));
-
- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
-
- // TODO: also execute sql queries.
-
- switch (rnd.nextInt(3)) {
- case 0:
- cache.getAndPut(key, v0);
- case 1:
- cache.get(key);
- break;
- case 2:
- cache.getAndRemove(key);
- }
- }
- }
-
- @Override
- protected void check(IgniteEx ig) {
- long pages = ig.context().cache().context().database().pageMemory().loadedPages();
-
- assertTrue(pages < 19100);
- }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e70d990f/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
index 2a6c8cd..8943743 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
@@ -18,13 +18,8 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
-
/**
*
*/
@@ -35,62 +30,29 @@ public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstra
static {
ARRAY = new int[1024];
- Random rnd = new Random();
-
for (int i = 0; i < ARRAY.length; i++)
- ARRAY[i] = rnd.nextInt();
- }
-
- @Override
- protected int duration() {
- return 300;
- }
-
- @Override
- protected int gridCount() {
- return 1;
+ ARRAY[i] = getRandom().nextInt();
}
- @Override
- protected void configure(IgniteConfiguration cfg) {
- cfg.setMetricsLogFrequency(5000);
+ /** {@inheritDoc} */
+ @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
+ return ig.cache("non-primitive");
}
- @Override
- protected void configure(MemoryConfiguration mCfg) {
- mCfg.setPageCacheSize(60 * 1024 * 1024);
+ /** {@inheritDoc} */
+ @Override protected Object key() {
+ return new DbKey(getRandom().nextInt(200_000));
}
- @Override
- protected boolean indexingEnabled() {
- return false;
- }
-
- protected void operation(IgniteEx ig){
- IgniteCache<Object, Object> cache = ig.cache("large");
- Random rnd = ThreadLocalRandom.current();
-
- for (int i = 0; i < 1000; i++) {
- LargeDbKey key = new LargeDbKey(rnd.nextInt(10_000), 1024);
-
- LargeDbValue v0 = new LargeDbValue("test-value-1-" + rnd.nextInt(200), "test-value-2-" + rnd.nextInt(200), ARRAY);
-
- switch (rnd.nextInt(3)) {
- case 0:
- cache.getAndPut(key, v0);
- case 1:
- cache.get(key);
- break;
- case 2:
- cache.getAndRemove(key);
- }
- }
+ /** {@inheritDoc} */
+ @Override protected Object value(Object key) {
+ return new DbValue(((DbKey)key).val, "test-value-" + getRandom().nextInt(200), getRandom().nextInt(500));
}
- @Override
- protected void check(IgniteEx ig) {
+ /** {@inheritDoc} */
+ @Override protected void check(IgniteEx ig) {
long pages = ig.context().cache().context().database().pageMemory().loadedPages();
- assertTrue(pages < 50000);
+ assertTrue(pages < 20000);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e70d990f/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
index 91c96af..8e4d0b4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
@@ -17,76 +17,31 @@
package org.apache.ignite.internal.processors.database;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
-
/**
*
*/
-public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakAbstractTest {
-
- @Override
- protected int duration() {
- return 300;
- }
+public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakTest {
- @Override
- protected int gridCount() {
- return 1;
- }
+ /** {@inheritDoc} */
+ @Override protected void configure(MemoryConfiguration mCfg) {
+ int concLvl = Runtime.getRuntime().availableProcessors();
+ mCfg.setConcurrencyLevel(concLvl);
+ mCfg.setPageCacheSize(1024 * 1024 * concLvl * 16);
- @Override
- protected void configure(IgniteConfiguration cfg) {
- cfg.setMetricsLogFrequency(5000);
}
- @Override
- protected void configure(MemoryConfiguration mCfg) {
- // TODO: understand why such overhead with large pages.
- mCfg.setPageCacheSize(100 * 1024 * 1024);
- }
-
- @Override
- protected boolean indexingEnabled() {
- return false;
- }
-
- @Override
- protected boolean isLargePage() {
+ /** {@inheritDoc} */
+ @Override protected boolean isLargePage() {
return true;
}
- // TODO: avoid copy/paste.
- protected void operation(IgniteEx ig){
- IgniteCache<Object, Object> cache = ig.cache("non-primitive");
- Random rnd = ThreadLocalRandom.current();
-
- for (int i = 0; i < 1000; i++) {
- DbKey key = new DbKey(rnd.nextInt(200_000));
-
- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
-
- switch (rnd.nextInt(3)) {
- case 0:
- cache.getAndPut(key, v0);
- case 1:
- cache.get(key);
- break;
- case 2:
- cache.getAndRemove(key);
- }
- }
- }
-
- @Override
- protected void check(IgniteEx ig) {
+ /** {@inheritDoc} */
+ @Override protected void check(IgniteEx ig) {
long pages = ig.context().cache().context().database().pageMemory().loadedPages();
- assertTrue(pages < 4600);
+ assertTrue(pages < 4000);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e70d990f/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
index 2b0ce1e..81d831b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
@@ -18,72 +18,31 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
-
/**
*
*/
public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
/** {@inheritDoc} */
- @Override protected int duration() {
- return 300;
- }
-
- @Override
- protected int gridCount() {
- return 1;
- }
-
- @Override
- protected void configure(IgniteConfiguration cfg) {
- cfg.setMetricsLogFrequency(5000);
- }
-
- @Override
- protected void configure(MemoryConfiguration mCfg) {
- mCfg.setPageCacheSize(1024 * 1024);
+ @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
+ return ig.cache("non-primitive");
}
- @Override
- protected boolean indexingEnabled() {
- return false;
+ /** {@inheritDoc} */
+ @Override protected Object key() {
+ return new DbKey(getRandom().nextInt(200_000));
}
- protected void operation(IgniteEx ig){
- IgniteCache<Object, Object> cache = ig.cache("non-primitive");
- Random rnd = ThreadLocalRandom.current();
-
- for (int i = 0; i < 1000; i++) {
- DbKey key = new DbKey(rnd.nextInt(200_000));
-
- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
-
- // TODO: also execute scan query.
-
- switch (rnd.nextInt(3)) {
- case 0:
- cache.getAndPut(key, v0);
- break;
-
- case 1:
- cache.get(key);
- break;
-
- case 2:
- cache.getAndRemove(key);
- }
- }
+ /** {@inheritDoc} */
+ @Override protected Object value(Object key) {
+ return new DbValue(((DbKey)key).val, "test-value-" + getRandom().nextInt(200), getRandom().nextInt(500));
}
- @Override
- protected void check(IgniteEx ig) {
+ /** {@inheritDoc} */
+ @Override protected void check(IgniteEx ig) {
long pages = ig.context().cache().context().database().pageMemory().loadedPages();
- assertTrue(pages < 19100);
+ assertTrue(pages < 20000);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e70d990f/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
index 95fe8c8..a31ffb4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
@@ -18,75 +18,31 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
import javax.cache.expiry.CreatedExpiryPolicy;
import javax.cache.expiry.Duration;
import javax.cache.expiry.ExpiryPolicy;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
/**
*
*/
-public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakAbstractTest {
+public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakTest {
/** */
private static final ExpiryPolicy EXPIRY = new CreatedExpiryPolicy(new Duration(MILLISECONDS, 10L));
- @Override
- protected int duration() {
- return 300;
+ /** {@inheritDoc} */
+ @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
+ return ig.cache("non-primitive").withExpiryPolicy(EXPIRY);
}
- @Override
- protected int gridCount() {
- return 1;
- }
-
- @Override
- protected void configure(IgniteConfiguration cfg) {
- cfg.setMetricsLogFrequency(5000);
- }
-
- @Override
- protected void configure(MemoryConfiguration mCfg) {
- mCfg.setPageCacheSize(1024 * 1024);
- }
-
- @Override
- protected boolean indexingEnabled() {
- return false;
- }
-
- protected void operation(IgniteEx ig) {
- IgniteCache<Object, Object> cache = ig.cache("non-primitive").withExpiryPolicy(EXPIRY);
- Random rnd = ThreadLocalRandom.current();
-
- for (int i = 0; i < 1000; i++) {
- DbKey key = new DbKey(rnd.nextInt(200_000));
-
- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
-
- switch (rnd.nextInt(3)) {
- case 0:
- cache.getAndPut(key, v0);
- case 1:
- cache.get(key);
- break;
- case 2:
- cache.getAndRemove(key);
- }
- }
- }
-
- @Override
- protected void check(IgniteEx ig) {
+ /** {@inheritDoc} */
+ @Override protected void check(IgniteEx ig) {
long pages = ig.context().cache().context().database().pageMemory().loadedPages();
- assertTrue(pages < 10000);
+ assertTrue(pages < 7000);
}
}
[33/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/af05cd8f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/af05cd8f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/af05cd8f
Branch: refs/heads/ignite-4712
Commit: af05cd8f0e7ab14db4bb19db40c6100627a1964b
Parents: 4e7912a
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Wed Feb 22 12:28:30 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:55 2017 +0300
----------------------------------------------------------------------
.../internal/processors/database/IgniteDbPutGetAbstractTest.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/af05cd8f/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
index 228a262..cf98dce 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
@@ -60,9 +60,7 @@ public abstract class IgniteDbPutGetAbstractTest extends IgniteDbAbstractTest {
final int cnt = 100_000;
- Random rnd = BPlusTree.rnd;
-
- assert rnd != null;
+ Random rnd = new Random();
Map<Integer, DbValue> map = new HashMap<>();
[43/43] ignite git commit: ignite-4712
Posted by sb...@apache.org.
ignite-4712
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8434edd6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8434edd6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8434edd6
Branch: refs/heads/ignite-4712
Commit: 8434edd61bc1cc83a669e373bdb2fe96ebc6c0a7
Parents: 40dbfbd
Author: sboikov <sb...@gridgain.com>
Authored: Fri Mar 10 10:13:01 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Mar 10 10:20:21 2017 +0300
----------------------------------------------------------------------
.../cache/database/freelist/PagesList.java | 11 +--
.../ignite/cache/LargeEntryUpdateTest.java | 97 ++++++++------------
.../database/IgniteDbAbstractTest.java | 1 -
.../IgniteDbMemoryLeakAbstractTest.java | 3 +-
.../database/IgniteDbPutGetAbstractTest.java | 22 ++---
.../ignite/testsuites/IgniteCacheTestSuite.java | 3 -
.../testsuites/IgniteDbMemoryLeakTestSuite.java | 3 +
7 files changed, 58 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8434edd6/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
index b2c6e9c..b2b4393 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
@@ -201,10 +201,9 @@ public abstract class PagesList extends DataStructure {
long tailId = upd[i];
long pageId = tailId;
- int count = 0;
+ int cnt = 0;
while (pageId != 0L) {
-
try (Page page = page(pageId)) {
long pageAddr = readLock(page);
@@ -213,12 +212,12 @@ public abstract class PagesList extends DataStructure {
try {
PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(pageAddr);
- count += io.getCount(pageAddr);
+ cnt += io.getCount(pageAddr);
pageId = io.getPreviousId(pageAddr);
// In reuse bucket the page itself can be used as a free page.
if (isReuseBucket(bucket) && pageId != 0L)
- count++;
+ cnt++;
}
finally {
readUnlock(page, pageAddr);
@@ -226,9 +225,9 @@ public abstract class PagesList extends DataStructure {
}
}
- Stripe stripe = new Stripe(tailId, count == 0);
+ Stripe stripe = new Stripe(tailId, cnt == 0);
tails[i] = stripe;
- bucketSize += count;
+ bucketSize += cnt;
}
boolean ok = casBucket(bucket, null, tails);
http://git-wip-us.apache.org/repos/asf/ignite/blob/8434edd6/modules/core/src/test/java/org/apache/ignite/cache/LargeEntryUpdateTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/LargeEntryUpdateTest.java b/modules/core/src/test/java/org/apache/ignite/cache/LargeEntryUpdateTest.java
index 6b63296..770c67d 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/LargeEntryUpdateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/LargeEntryUpdateTest.java
@@ -16,35 +16,34 @@
*/
package org.apache.ignite.cache;
+import java.util.ArrayList;
+import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
-import javax.cache.processor.EntryProcessorException;
import javax.cache.processor.MutableEntry;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteException;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.internal.mem.OutOfMemoryException;
+import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgniteRunnable;
import org.apache.ignite.resources.IgniteInstanceResource;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
/**
- * LargeEntryUpdateTest
*
- * @author Alexandr Kuramshin <ei...@gmail.com>
*/
public class LargeEntryUpdateTest extends GridCommonAbstractTest {
-
/** */
private static final int THREAD_COUNT = 10;
/** */
- private static final int PAGE_SIZE = 1 << 10; // 1 kB
+ private static final int PAGE_SIZE = 1 << 10; // 1 kB.
/** */
- private static final int PAGE_CACHE_SIZE = 30 << 20; // 30 MB
+ private static final int PAGE_CACHE_SIZE = 30 << 20; // 30 MB.
/** */
private static final String CACHE_PREFIX = "testCache";
@@ -53,17 +52,14 @@ public class LargeEntryUpdateTest extends GridCommonAbstractTest {
private static final int CACHE_COUNT = 10;
/** */
- private static final long WAIT_TIMEOUT = 5 * 60_000L; // 5 min
+ private static final long WAIT_TIMEOUT = 5 * 60_000L; // 5 min.
/** */
- private static final long TEST_TIMEOUT = 10 * 60_000L; // 10 min
+ private static final long TEST_TIMEOUT = 10 * 60_000L; // 10 min.
/** */
private final AtomicBoolean cacheUpdate = new AtomicBoolean();
- /** */
- private OutOfMemoryException outOfMemoryEx;
-
/** {@inheritDoc} */
@Override protected long getTestTimeout() {
return TEST_TIMEOUT;
@@ -72,14 +68,18 @@ public class LargeEntryUpdateTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(gridName);
+
cfg.setPublicThreadPoolSize(THREAD_COUNT);
MemoryConfiguration mem = new MemoryConfiguration();
+
mem.setPageSize(PAGE_SIZE);
mem.setPageCacheSize(PAGE_CACHE_SIZE);
+
cfg.setMemoryConfiguration(mem);
CacheConfiguration[] ccfgs = new CacheConfiguration[CACHE_COUNT];
+
for (int i = 0; i < CACHE_COUNT; ++i) {
CacheConfiguration ccfg = new CacheConfiguration();
ccfg.setName(CACHE_PREFIX + i);
@@ -87,33 +87,42 @@ public class LargeEntryUpdateTest extends GridCommonAbstractTest {
ccfg.setCacheMode(CacheMode.PARTITIONED);
ccfgs[i] = ccfg;
}
+
cfg.setCacheConfiguration(ccfgs);
return cfg;
}
- /** */
+ /**
+ * @throws Exception If failed.
+ */
public void testEntryUpdate() throws Exception {
try (Ignite ignite = startGrid()) {
for (int i = 0; i < CACHE_COUNT; ++i) {
IgniteCache<Long, byte[]> cache = ignite.cache(CACHE_PREFIX + i);
+
cache.put(0L, new byte[PAGE_SIZE * 2]);
}
+
cacheUpdate.set(true);
+
IgniteCompute compute = ignite.compute().withAsync();
- for (int i = 0; i < THREAD_COUNT; ++i)
- compute.run(new CacheUpdater());
+
+ long endTime = System.currentTimeMillis() + WAIT_TIMEOUT;
+
try {
- long deadline = System.currentTimeMillis() + WAIT_TIMEOUT;
- while (true)
- synchronized (this) {
- if (outOfMemoryEx != null)
- throw outOfMemoryEx;
- long timeout = deadline - System.currentTimeMillis();
- if (timeout <= 0)
- break;
- wait(timeout);
+ while (System.currentTimeMillis() < endTime) {
+ List<IgniteFuture> futs = new ArrayList<>();
+
+ for (int i = 0; i < THREAD_COUNT; ++i) {
+ compute.run(new CacheUpdater());
+
+ futs.add(compute.future());
}
+
+ for (IgniteFuture fut : futs)
+ fut.get();
+ }
}
finally {
cacheUpdate.set(false);
@@ -122,29 +131,20 @@ public class LargeEntryUpdateTest extends GridCommonAbstractTest {
}
/** */
- private synchronized void setOutOfMemoryEx(OutOfMemoryException e) {
- outOfMemoryEx = e;
- notifyAll();
- }
-
- /** */
public static class EntryUpdater implements CacheEntryProcessor<Long, byte[], Void> {
-
/** */
public static final EntryUpdater INSTANCE = new EntryUpdater();
/** {@inheritDoc} */
- @Override public Void process(MutableEntry<Long, byte[]> entry, Object... arguments)
- throws EntryProcessorException {
-
+ @Override public Void process(MutableEntry<Long, byte[]> entry, Object... args) {
entry.setValue(new byte[PAGE_SIZE]);
+
return null;
}
}
/** */
public class CacheUpdater implements IgniteRunnable {
-
/** */
@IgniteInstanceResource
public transient Ignite ignite;
@@ -152,37 +152,18 @@ public class LargeEntryUpdateTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override public void run() {
try {
- while (cacheUpdate.get())
+ while (cacheUpdate.get()) {
for (int i = 0; i < CACHE_COUNT; ++i) {
IgniteCache<Long, byte[]> cache = ignite.cache(CACHE_PREFIX + i);
+
cache.invoke(0L, EntryUpdater.INSTANCE);
}
+ }
}
catch (Throwable ex) {
- OutOfMemoryException e = findOutOfMemoryException(ex);
- if (e != null)
- setOutOfMemoryEx(e);
- else
- ignite.log().error("CacheUpdater failed", ex);
+ throw new IgniteException(ex);
}
}
-
- /** */
- private OutOfMemoryException findOutOfMemoryException(Throwable ex) {
- if (ex instanceof OutOfMemoryException)
- return (OutOfMemoryException)ex;
- for (Throwable s : ex.getSuppressed()) {
- OutOfMemoryException e = findOutOfMemoryException(s);
- if (e != null)
- return e;
- }
- if (ex.getCause() != null) {
- OutOfMemoryException e = findOutOfMemoryException(ex.getCause());
- if (e != null)
- return e;
- }
- return null;
- }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8434edd6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
index 46efab2..cf26187 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
@@ -68,7 +68,6 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
else
dbCfg.setPageSize(1024);
-
dbCfg.setPageCacheSize(200 * 1024 * 1024);
configure(dbCfg);
http://git-wip-us.apache.org/repos/asf/ignite/blob/8434edd6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index 06ecb94..2e9d3f9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -172,9 +172,8 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
*/
public void testMemoryLeak() throws Exception {
final IgniteEx ignite = grid(0);
- final IgniteCache<Object, Object> cache = cache(ignite);
-
+ final IgniteCache<Object, Object> cache = cache(ignite);
Runnable target = new Runnable() {
@Override public void run() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/8434edd6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
index cf98dce..12b0126 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
@@ -17,6 +17,16 @@
package org.apache.ignite.internal.processors.database;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import javax.cache.Cache;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteDataStreamer;
@@ -28,24 +38,12 @@ import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.cache.query.SqlQuery;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
-import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
import org.apache.ignite.internal.util.GridRandom;
import org.apache.ignite.internal.util.typedef.PA;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.testframework.GridTestUtils;
import org.junit.Assert;
-import javax.cache.Cache;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ThreadLocalRandom;
-
/**
*
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/8434edd6/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index efa3225..1cf2c14 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -22,7 +22,6 @@ import junit.framework.TestSuite;
import org.apache.ignite.GridCacheAffinityBackupsSelfTest;
import org.apache.ignite.IgniteCacheAffinitySelfTest;
import org.apache.ignite.cache.IgniteWarmupClosureSelfTest;
-import org.apache.ignite.cache.LargeEntryUpdateTest;
import org.apache.ignite.cache.affinity.AffinityClientNodeSelfTest;
import org.apache.ignite.cache.affinity.AffinityHistoryCleanupTest;
import org.apache.ignite.cache.affinity.fair.FairAffinityDynamicCacheSelfTest;
@@ -230,8 +229,6 @@ public class IgniteCacheTestSuite extends TestSuite {
// Warmup closure tests.
suite.addTestSuite(IgniteWarmupClosureSelfTest.class);
- suite.addTestSuite(LargeEntryUpdateTest.class);
-
// Swap tests.
suite.addTestSuite(GridCacheSwapPreloadSelfTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/8434edd6/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakTestSuite.java
index 75b31b0..f271bd8 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakTestSuite.java
@@ -18,6 +18,7 @@
package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
+import org.apache.ignite.cache.LargeEntryUpdateTest;
import org.apache.ignite.internal.processors.database.IgniteDbMemoryLeakLargeObjectsTest;
import org.apache.ignite.internal.processors.database.IgniteDbMemoryLeakLargePagesTest;
import org.apache.ignite.internal.processors.database.IgniteDbMemoryLeakNonTransactionalTest;
@@ -41,6 +42,8 @@ public class IgniteDbMemoryLeakTestSuite extends TestSuite {
suite.addTestSuite(IgniteDbMemoryLeakLargeObjectsTest.class);
suite.addTestSuite(IgniteDbMemoryLeakNonTransactionalTest.class);
+ suite.addTestSuite(LargeEntryUpdateTest.class);
+
return suite;
}
}
[08/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/fa395add
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/fa395add
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/fa395add
Branch: refs/heads/ignite-4712
Commit: fa395addf03376423b9aeb15dd74a7a9969cb851
Parents: e259b7a
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Mon Feb 20 13:43:36 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Tue Feb 21 12:44:29 2017 +0300
----------------------------------------------------------------------
.../processors/database/IgniteDbMemoryLeakAbstractTest.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/fa395add/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index 819405e..d939311 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -23,6 +23,7 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.database.DataStructure;
import org.apache.ignite.internal.util.GridRandom;
import org.jetbrains.annotations.NotNull;
@@ -55,6 +56,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
@Override protected void beforeTest() throws Exception {
super.beforeTest();
+ DataStructure.rnd = null;
long startTime = System.nanoTime();
warmUpEndTime = startTime + TimeUnit.SECONDS.toNanos(warmUp());
@@ -246,6 +248,6 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
* @return Expected average number of pages, on which their total number can grow per 5 seconds.
*/
@SuppressWarnings("WeakerAccess") protected int pagesDelta() {
- return 5;
+ return 3;
}
}
[26/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6e56fa2a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6e56fa2a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6e56fa2a
Branch: refs/heads/ignite-4712
Commit: 6e56fa2a8eeb019f063da51df73e8922a114ba3d
Parents: b3d78b5
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Mon Feb 20 16:40:25 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:54 2017 +0300
----------------------------------------------------------------------
.../cache/database/freelist/PagesList.java | 30 ++++++++++++++++++--
1 file changed, 28 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6e56fa2a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
index 1f537b6..5ae3549 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
@@ -189,6 +189,7 @@ public abstract class PagesList extends DataStructure {
for (Map.Entry<Integer, GridLongList> e : bucketsData.entrySet()) {
int bucket = e.getKey();
+ long bucketSize = 0;
Stripe[] old = getBucket(bucket);
assert old == null;
@@ -197,11 +198,36 @@ public abstract class PagesList extends DataStructure {
Stripe[] tails = new Stripe[upd.length];
- for (int i = 0; i < upd.length; i++)
- tails[i] = new Stripe(upd[i]);
+ for (int i = 0; i < upd.length; i++) {
+ long tailId = upd[i];
+
+ try(Page tail = page(tailId)) {
+ long tailAddr = readLock(tail);
+
+ assert tailAddr != 0L;
+
+ try {
+ PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(tailAddr);
+
+ int count = io.getCount(tailAddr);
+
+ Stripe stripe = new Stripe(tailId);
+ stripe.empty = count == 0;
+
+ tails[i] = stripe;
+
+ bucketSize += count;
+ }
+ finally {
+ readUnlock(tail, tailAddr);
+ }
+ }
+ }
boolean ok = casBucket(bucket, null, tails);
assert ok;
+
+ bucketsSize[bucket].set(bucketSize);
}
}
}
[30/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1678e1b9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1678e1b9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1678e1b9
Branch: refs/heads/ignite-4712
Commit: 1678e1b9fd79868c5246533189a8f619eb6033a1
Parents: 0ee1165
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Mon Feb 20 18:31:15 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:54 2017 +0300
----------------------------------------------------------------------
.../processors/database/IgniteDbMemoryLeakAbstractTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1678e1b9/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index bef1320..ccb1fa0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -103,7 +103,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/** {@inheritDoc} */
@Override protected long getTestTimeout() {
- return (warmUp() + duration() + 1) * 1000; // One extra second to stop all threads
+ return (warmUp() + duration() + 1) * 2000; // Two extra seconds to stop all threads.
}
/**
[28/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/21d62829
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/21d62829
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/21d62829
Branch: refs/heads/ignite-4712
Commit: 21d6282953b47c94559b708b343172ca0049e984
Parents: 6743d5c
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Mon Feb 20 13:40:05 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:54 2017 +0300
----------------------------------------------------------------------
.../cache/database/freelist/PagesList.java | 39 ++++++++++++--------
.../IgniteDbMemoryLeakLargeObjectsTest.java | 14 ++++---
2 files changed, 32 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/21d62829/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
index e5430cf..c03f432 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
@@ -76,9 +76,6 @@ public abstract class PagesList extends DataStructure {
Math.min(8, Runtime.getRuntime().availableProcessors() * 2));
/** */
- private final boolean trackBucketsSize = IgniteSystemProperties.getBoolean("IGNITE_PAGES_LIST_TRACK_SIZE", false);
-
- /** */
protected final LongAdder8[] bucketsSize;
/** Page ID to store list metadata. */
@@ -600,8 +597,7 @@ public abstract class PagesList extends DataStructure {
if (idx == -1)
handlePageFull(pageId, page, pageAddr, io, dataPage, dataPageAddr, bucket);
else {
- if (trackBucketsSize)
- bucketsSize[bucket].increment();
+ bucketsSize[bucket].increment();
if (isWalDeltaRecordNeeded(wal, page))
wal.log(new PagesListAddPageRecord(cacheId, pageId, dataPageId));
@@ -695,8 +691,7 @@ public abstract class PagesList extends DataStructure {
assert idx != -1;
- if (trackBucketsSize)
- bucketsSize[bucket].increment();
+ bucketsSize[bucket].increment();
dataIO.setFreeListPageId(dataPageAddr, nextId);
@@ -786,8 +781,7 @@ public abstract class PagesList extends DataStructure {
}
}
else {
- if (trackBucketsSize)
- bucketsSize[bucket].increment();
+ bucketsSize[bucket].increment();
// TODO: use single WAL record for bag?
if (isWalDeltaRecordNeeded(wal, page))
@@ -819,7 +813,18 @@ public abstract class PagesList extends DataStructure {
if (tails == null)
return null;
- return randomTail(tails);
+ int len = tails.length;
+ int init = randomInt(len);
+ int cur = init;
+
+ while (true) {
+ Stripe stripe = tails[cur];
+ if(!stripe.empty)
+ return stripe;
+
+ if((cur = (cur + 1) % len) == init)
+ return null;
+ }
}
/**
@@ -873,7 +878,7 @@ public abstract class PagesList extends DataStructure {
for (int lockAttempt = 0; ;) {
Stripe stripe = getPageForTake(bucket);
- if (stripe == null || stripe.empty)
+ if (stripe == null)
return 0L;
long tailId = stripe.tailId;
@@ -904,8 +909,7 @@ public abstract class PagesList extends DataStructure {
long pageId = io.takeAnyPage(tailPageAddr);
if (pageId != 0L) {
- if (trackBucketsSize)
- bucketsSize[bucket].decrement();
+ bucketsSize[bucket].decrement();
if (isWalDeltaRecordNeeded(wal, tail))
wal.log(new PagesListRemovePageRecord(cacheId, tailId, pageId));
@@ -965,8 +969,12 @@ public abstract class PagesList extends DataStructure {
ret = tailId;
}
- else
+ else {
stripe.empty = true;
+
+ if (bucketsSize[bucket].sum() > 0)
+ continue;
+ }
}
// If we do not have a previous page (we are at head), then we still can return
@@ -1026,8 +1034,7 @@ public abstract class PagesList extends DataStructure {
if (!rmvd)
return false;
- if (trackBucketsSize)
- bucketsSize[bucket].decrement();
+ bucketsSize[bucket].decrement();
if (isWalDeltaRecordNeeded(wal, page))
wal.log(new PagesListRemovePageRecord(cacheId, pageId, dataPageId));
http://git-wip-us.apache.org/repos/asf/ignite/blob/21d62829/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
index c1b1c9b..0fa9096 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
/**
@@ -35,6 +36,13 @@ public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstra
}
/** {@inheritDoc} */
+ @Override protected void configure(MemoryConfiguration mCfg) {
+ super.configure(mCfg);
+
+ mCfg.setPageCacheSize(35840000); // The space for 35000 pages
+ }
+
+ /** {@inheritDoc} */
@Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
return ig.cache("large");
}
@@ -49,12 +57,8 @@ public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstra
return new LargeDbValue("test-value-1-" + getRandom().nextInt(200), "test-value-2-" + getRandom().nextInt(200), ARRAY);
}
- @Override protected int warmUp() {
- return 600;
- }
-
/** {@inheritDoc} */
@Override protected long pagesMax() {
- return 2000000;
+ return 35000;
}
}
[29/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5e11f5d2
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5e11f5d2
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5e11f5d2
Branch: refs/heads/ignite-4712
Commit: 5e11f5d248f8e8bb11f9f317f75762cd78553584
Parents: 21d6282
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Mon Feb 20 13:43:36 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:54 2017 +0300
----------------------------------------------------------------------
.../processors/database/IgniteDbMemoryLeakAbstractTest.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e11f5d2/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index 819405e..d939311 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -23,6 +23,7 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.database.DataStructure;
import org.apache.ignite.internal.util.GridRandom;
import org.jetbrains.annotations.NotNull;
@@ -55,6 +56,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
@Override protected void beforeTest() throws Exception {
super.beforeTest();
+ DataStructure.rnd = null;
long startTime = System.nanoTime();
warmUpEndTime = startTime + TimeUnit.SECONDS.toNanos(warmUp());
@@ -246,6 +248,6 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
* @return Expected average number of pages, on which their total number can grow per 5 seconds.
*/
@SuppressWarnings("WeakerAccess") protected int pagesDelta() {
- return 5;
+ return 3;
}
}
[10/43] ignite git commit: IGNITE-4694 Add tests to check there are
no memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4694 Add tests to check there are no memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5e4e9913
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5e4e9913
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5e4e9913
Branch: refs/heads/ignite-4712
Commit: 5e4e991360c2f2917ec22128e40a353c13db8285
Parents: 4ae7d71
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Tue Feb 21 11:49:35 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Tue Feb 21 12:44:30 2017 +0300
----------------------------------------------------------------------
.../IgniteDbMemoryLeakAbstractTest.java | 19 ++++--
.../IgniteDbMemoryLeakLargeObjectsTest.java | 8 +--
.../database/IgniteDbMemoryLeakTest.java | 4 +-
.../IgniteDbMemoryLeakWithExpirationTest.java | 1 +
.../testsuites/IgniteDbMemoryLeakTestSuite.java | 46 +++++++++++++
.../IgniteDbMemoryLeakSqlQueryTest.java | 71 ++++++++++++++++++++
...IgniteDbMemoryLeakWithIndexingTestSuite.java | 40 +++++++++++
7 files changed, 177 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e4e9913/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index ccb1fa0..93306d9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -17,7 +17,6 @@
package org.apache.ignite.internal.processors.database;
-import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import org.apache.ignite.IgniteCache;
@@ -25,7 +24,6 @@ import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.processors.cache.database.DataStructure;
-import org.jetbrains.annotations.NotNull;
/**
* Base class for memory leaks tests.
@@ -55,6 +53,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/** */
private long probeCnt = 0;
+ /** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
super.beforeTest();
DataStructure.rnd = null;
@@ -130,7 +129,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
Object key = key();
Object value = value(key);
- switch (getRandom().nextInt(3)) {
+ switch (nextInt(3)) {
case 0:
cache.getAndPut(key, value);
@@ -145,10 +144,18 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
}
/**
- * @return Random.
+ * @param bound Upper bound (exclusive). Must be positive.
+ * @return Random int value.
+ */
+ protected static int nextInt(int bound) {
+ return ThreadLocalRandom.current().nextInt(bound);
+ }
+
+ /**
+ * @return Random int value.
*/
- @NotNull protected static Random getRandom() {
- return ThreadLocalRandom.current();
+ protected static int nextInt() {
+ return ThreadLocalRandom.current().nextInt();
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e4e9913/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
index 3aab102..077a1e1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
@@ -31,7 +31,7 @@ public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstra
ARRAY = new int[1024];
for (int i = 0; i < ARRAY.length; i++)
- ARRAY[i] = getRandom().nextInt();
+ ARRAY[i] = nextInt();
}
/** {@inheritDoc} */
@@ -41,16 +41,16 @@ public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstra
/** {@inheritDoc} */
@Override protected Object key() {
- return new LargeDbKey(getRandom().nextInt(10_000), 1024);
+ return new LargeDbKey(nextInt(10_000), 1024);
}
/** {@inheritDoc} */
@Override protected Object value(Object key) {
- return new LargeDbValue("test-value-1-" + getRandom().nextInt(200), "test-value-2-" + getRandom().nextInt(200), ARRAY);
+ return new LargeDbValue("test-value-1-" + nextInt(200), "test-value-2-" + nextInt(200), ARRAY);
}
/** {@inheritDoc} */
@Override protected long pagesMax() {
- return 35000;
+ return 35_000;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e4e9913/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
index 30ff18f..b8ac8f0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
@@ -31,12 +31,12 @@ public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
/** {@inheritDoc} */
@Override protected Object key() {
- return new DbKey(getRandom().nextInt(200_000));
+ return new DbKey(nextInt(200_000));
}
/** {@inheritDoc} */
@Override protected Object value(Object key) {
- return new DbValue(((DbKey)key).val, "test-value-" + getRandom().nextInt(200), getRandom().nextInt(500));
+ return new DbValue(((DbKey)key).val, "test-value-" + nextInt(200), nextInt(500));
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e4e9913/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
index 212daa1..19502b0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
@@ -38,6 +38,7 @@ public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakTest
return ig.cache("non-primitive").withExpiryPolicy(EXPIRY);
}
+ /** {@inheritDoc} */
@Override protected long pagesMax() {
return 7000;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e4e9913/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakTestSuite.java
new file mode 100644
index 0000000..75b31b0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakTestSuite.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.database.IgniteDbMemoryLeakLargeObjectsTest;
+import org.apache.ignite.internal.processors.database.IgniteDbMemoryLeakLargePagesTest;
+import org.apache.ignite.internal.processors.database.IgniteDbMemoryLeakNonTransactionalTest;
+import org.apache.ignite.internal.processors.database.IgniteDbMemoryLeakTest;
+import org.apache.ignite.internal.processors.database.IgniteDbMemoryLeakWithExpirationTest;
+
+/**
+ * Page memory leaks tests.
+ */
+public class IgniteDbMemoryLeakTestSuite extends TestSuite {
+ /**
+ * @return Test suite.
+ * @throws Exception Thrown in case of the failure.
+ */
+ public static TestSuite suite() throws Exception {
+ TestSuite suite = new TestSuite("Ignite Db Memory Leaks Test Suite");
+
+ suite.addTestSuite(IgniteDbMemoryLeakTest.class);
+ suite.addTestSuite(IgniteDbMemoryLeakWithExpirationTest.class);
+ suite.addTestSuite(IgniteDbMemoryLeakLargePagesTest.class);
+ suite.addTestSuite(IgniteDbMemoryLeakLargeObjectsTest.class);
+ suite.addTestSuite(IgniteDbMemoryLeakNonTransactionalTest.class);
+
+ return suite;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e4e9913/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakSqlQueryTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakSqlQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakSqlQueryTest.java
new file mode 100644
index 0000000..b65cac0
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakSqlQueryTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ *
+ */
+public class IgniteDbMemoryLeakSqlQueryTest extends IgniteDbMemoryLeakTest {
+ /** {@inheritDoc} */
+ @Override protected boolean indexingEnabled() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long pagesMax() {
+ return 24_000;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void operation(IgniteCache<Object, Object> cache) {
+ Object key = key();
+ Object value = value(key);
+
+ switch (nextInt(4)) {
+ case 0:
+ cache.getAndPut(key, value);
+
+ break;
+ case 1:
+ cache.get(key);
+
+ break;
+ case 2:
+ cache.getAndRemove(key);
+
+ break;
+ case 3:
+ cache.query(sqlQuery(cache)).getAll();
+ }
+ }
+
+ /**
+ * @param cache IgniteCache.
+ * @return SqlFieldsQuery.
+ */
+ @NotNull private SqlFieldsQuery sqlQuery(IgniteCache<Object, Object> cache) {
+ String query = String.format("select _key from \"%s\".DbValue where iVal=?", cache.getName());
+ SqlFieldsQuery sqlQuery = new SqlFieldsQuery(query);
+ sqlQuery.setArgs(nextInt(200_000));
+ return sqlQuery;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e4e9913/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakWithIndexingTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakWithIndexingTestSuite.java
new file mode 100644
index 0000000..36cd101
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakWithIndexingTestSuite.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.database.IgniteDbMemoryLeakIndexedTest;
+import org.apache.ignite.internal.processors.database.IgniteDbMemoryLeakSqlQueryTest;
+
+/**
+ * Page memory leaks tests using indexing.
+ */
+public class IgniteDbMemoryLeakWithIndexingTestSuite extends TestSuite {
+ /**
+ * @return Test suite.
+ * @throws Exception Thrown in case of the failure.
+ */
+ public static TestSuite suite() throws Exception {
+ TestSuite suite = new TestSuite("Ignite Db Memory Leaks With Indexing Test Suite");
+
+ suite.addTestSuite(IgniteDbMemoryLeakSqlQueryTest.class);
+ suite.addTestSuite(IgniteDbMemoryLeakIndexedTest.class);
+
+ return suite;
+ }
+}
[19/43] ignite git commit: IGNITE-4694 Add tests to check there are
no memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4694 Add tests to check there are no memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a2ae747d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a2ae747d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a2ae747d
Branch: refs/heads/ignite-4712
Commit: a2ae747dff48515b8e1d968f56a1cf93188a905e
Parents: 4c3ccb3
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Wed Feb 22 15:01:09 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Feb 22 15:01:09 2017 +0300
----------------------------------------------------------------------
.../processors/database/IgniteDbMemoryLeakAbstractTest.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a2ae747d/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index 04ac5ea..06ecb94 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -25,6 +25,8 @@ import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.processors.cache.database.DataStructure;
+import static org.apache.ignite.IgniteSystemProperties.getInteger;
+
/**
* Base class for memory leaks tests.
*/
@@ -84,7 +86,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
* @return Test duration in seconds.
*/
protected int duration() {
- return 300;
+ return getInteger("IGNITE_MEMORY_LEAKS_TEST_DURATION", 300);
}
/**
@@ -92,7 +94,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
*/
@SuppressWarnings("WeakerAccess")
protected int warmUp() {
- return 450;
+ return getInteger("IGNITE_MEMORY_LEAKS_TEST_WARM_UP", 450);
}
/** {@inheritDoc} */
[38/43] ignite git commit: ignite-4712
Posted by sb...@apache.org.
ignite-4712
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7d6bf32d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7d6bf32d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7d6bf32d
Branch: refs/heads/ignite-4712
Commit: 7d6bf32d0251cf14d0f834a9af09ea9ee691a83f
Parents: cae3a22
Author: sboikov <sb...@gridgain.com>
Authored: Wed Feb 22 11:15:04 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:55 2017 +0300
----------------------------------------------------------------------
.../cache/database/freelist/PagesList.java | 59 ++++++++++++++++----
1 file changed, 47 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/7d6bf32d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
index a5fb617..5fdd191 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
@@ -52,7 +52,6 @@ import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.jetbrains.annotations.Nullable;
-import org.jsr166.LongAdder8;
import static java.lang.Boolean.FALSE;
import static java.lang.Boolean.TRUE;
@@ -201,6 +200,7 @@ public abstract class PagesList extends DataStructure {
for (int i = 0; i < upd.length; i++) {
long tailId = upd[i];
+ // TODO: just relase readlock.
List<Page> locked = new ArrayList<>(2);
List<Long> lockedAddrs = new ArrayList<>(2);
@@ -222,14 +222,13 @@ public abstract class PagesList extends DataStructure {
count += io.getCount(pageAddr);
pageId = io.getPreviousId(pageAddr);
- // In reuse bucket the page itself can be used as a free page
- if(isReuseBucket(bucket) && pageId != 0L)
+ // In reuse bucket the page itself can be used as a free page.
+ if (isReuseBucket(bucket) && pageId != 0L)
count++;
}
}
- Stripe stripe = new Stripe(tailId);
- stripe.empty = count == 0;
+ Stripe stripe = new Stripe(tailId, count == 0);
tails[i] = stripe;
@@ -405,8 +404,7 @@ public abstract class PagesList extends DataStructure {
initPage(pageMem, page, this, PagesListNodeIO.VERSIONS.latest(), wal);
}
- Stripe stripe = new Stripe(pageId);
- stripe.empty = true;
+ Stripe stripe = new Stripe(pageId, true);
for (;;) {
Stripe[] old = getBucket(bucket);
@@ -518,11 +516,43 @@ public abstract class PagesList extends DataStructure {
}
/**
+ * !!! For tests only, does not provide any correctness guarantees for concurrent access.
+ *
* @param bucket Bucket index.
* @return Number of pages stored in this list.
+ * @throws IgniteCheckedException If failed.
*/
- protected final long storedPagesCount(int bucket) {
- return bucketsSize[bucket].get();
+ protected final long storedPagesCount(int bucket) throws IgniteCheckedException {
+ long res = 0;
+
+ Stripe[] tails = getBucket(bucket);
+
+ if (tails != null) {
+ for (Stripe tail : tails) {
+ long pageId = tail.tailId;
+
+ try (Page page = page(pageId)) {
+ long pageAddr = readLock(page); // No correctness guaranties.
+
+ try {
+ PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(pageAddr);
+
+ int cnt = io.getCount(pageAddr);
+
+ assert cnt >= 0;
+
+ res += cnt;
+ }
+ finally {
+ readUnlock(page, pageAddr);
+ }
+ }
+ }
+ }
+
+ assert res == bucketsSize[bucket].get();
+
+ return res;
}
/**
@@ -672,6 +702,7 @@ public abstract class PagesList extends DataStructure {
dataPageId,
pageId, 0L));
+ // In reuse bucket the page itself can be used as a free page.
bucketsSize[bucket].incrementAndGet();
updateTail(bucket, pageId, dataPageId);
@@ -791,7 +822,8 @@ public abstract class PagesList extends DataStructure {
0L
));
- if(isReuseBucket(bucket))
+ // In reuse bucket the page itself can be used as a free page.
+ if (isReuseBucket(bucket))
bucketsSize[bucket].incrementAndGet();
// Switch to this new page, which is now a part of our list
@@ -938,9 +970,10 @@ public abstract class PagesList extends DataStructure {
try {
PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(tailPageAddr);
- if (io.getNextId(tailPageAddr) != 0)
+ if (io.getNextId(tailPageAddr) != 0) {
// It is not a tail anymore, retry.
continue;
+ }
long pageId = io.takeAnyPage(tailPageAddr);
@@ -1356,9 +1389,11 @@ public abstract class PagesList extends DataStructure {
/**
* @param tailId Tail ID.
+ * @param empty Empty flag.
*/
- Stripe(long tailId) {
+ Stripe(long tailId, boolean empty) {
this.tailId = tailId;
+ this.empty = empty;
}
}
}
[32/43] ignite git commit: [GG-11997] OutOfMemoryException when
repeatedly updating the same entry larger than (page size / 2)
Posted by sb...@apache.org.
[GG-11997] OutOfMemoryException when repeatedly updating the same entry larger than (page size / 2)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/869089db
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/869089db
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/869089db
Branch: refs/heads/ignite-4712
Commit: 869089db542894b7bdbf4f87b09131cfcd086dd8
Parents: b2f1ea9
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Tue Feb 28 14:28:11 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:55 2017 +0300
----------------------------------------------------------------------
.../ignite/cache/LargeEntryUpdateTest.java | 188 +++++++++++++++++++
.../ignite/testsuites/IgniteCacheTestSuite.java | 3 +
2 files changed, 191 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/869089db/modules/core/src/test/java/org/apache/ignite/cache/LargeEntryUpdateTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/LargeEntryUpdateTest.java b/modules/core/src/test/java/org/apache/ignite/cache/LargeEntryUpdateTest.java
new file mode 100644
index 0000000..6b63296
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/LargeEntryUpdateTest.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.cache;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.internal.mem.OutOfMemoryException;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * LargeEntryUpdateTest
+ *
+ * @author Alexandr Kuramshin <ei...@gmail.com>
+ */
+public class LargeEntryUpdateTest extends GridCommonAbstractTest {
+
+ /** */
+ private static final int THREAD_COUNT = 10;
+
+ /** */
+ private static final int PAGE_SIZE = 1 << 10; // 1 kB
+
+ /** */
+ private static final int PAGE_CACHE_SIZE = 30 << 20; // 30 MB
+
+ /** */
+ private static final String CACHE_PREFIX = "testCache";
+
+ /** */
+ private static final int CACHE_COUNT = 10;
+
+ /** */
+ private static final long WAIT_TIMEOUT = 5 * 60_000L; // 5 min
+
+ /** */
+ private static final long TEST_TIMEOUT = 10 * 60_000L; // 10 min
+
+ /** */
+ private final AtomicBoolean cacheUpdate = new AtomicBoolean();
+
+ /** */
+ private OutOfMemoryException outOfMemoryEx;
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return TEST_TIMEOUT;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+ cfg.setPublicThreadPoolSize(THREAD_COUNT);
+
+ MemoryConfiguration mem = new MemoryConfiguration();
+ mem.setPageSize(PAGE_SIZE);
+ mem.setPageCacheSize(PAGE_CACHE_SIZE);
+ cfg.setMemoryConfiguration(mem);
+
+ CacheConfiguration[] ccfgs = new CacheConfiguration[CACHE_COUNT];
+ for (int i = 0; i < CACHE_COUNT; ++i) {
+ CacheConfiguration ccfg = new CacheConfiguration();
+ ccfg.setName(CACHE_PREFIX + i);
+ ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+ ccfg.setCacheMode(CacheMode.PARTITIONED);
+ ccfgs[i] = ccfg;
+ }
+ cfg.setCacheConfiguration(ccfgs);
+
+ return cfg;
+ }
+
+ /** */
+ public void testEntryUpdate() throws Exception {
+ try (Ignite ignite = startGrid()) {
+ for (int i = 0; i < CACHE_COUNT; ++i) {
+ IgniteCache<Long, byte[]> cache = ignite.cache(CACHE_PREFIX + i);
+ cache.put(0L, new byte[PAGE_SIZE * 2]);
+ }
+ cacheUpdate.set(true);
+ IgniteCompute compute = ignite.compute().withAsync();
+ for (int i = 0; i < THREAD_COUNT; ++i)
+ compute.run(new CacheUpdater());
+ try {
+ long deadline = System.currentTimeMillis() + WAIT_TIMEOUT;
+ while (true)
+ synchronized (this) {
+ if (outOfMemoryEx != null)
+ throw outOfMemoryEx;
+ long timeout = deadline - System.currentTimeMillis();
+ if (timeout <= 0)
+ break;
+ wait(timeout);
+ }
+ }
+ finally {
+ cacheUpdate.set(false);
+ }
+ }
+ }
+
+ /** */
+ private synchronized void setOutOfMemoryEx(OutOfMemoryException e) {
+ outOfMemoryEx = e;
+ notifyAll();
+ }
+
+ /** */
+ public static class EntryUpdater implements CacheEntryProcessor<Long, byte[], Void> {
+
+ /** */
+ public static final EntryUpdater INSTANCE = new EntryUpdater();
+
+ /** {@inheritDoc} */
+ @Override public Void process(MutableEntry<Long, byte[]> entry, Object... arguments)
+ throws EntryProcessorException {
+
+ entry.setValue(new byte[PAGE_SIZE]);
+ return null;
+ }
+ }
+
+ /** */
+ public class CacheUpdater implements IgniteRunnable {
+
+ /** */
+ @IgniteInstanceResource
+ public transient Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ try {
+ while (cacheUpdate.get())
+ for (int i = 0; i < CACHE_COUNT; ++i) {
+ IgniteCache<Long, byte[]> cache = ignite.cache(CACHE_PREFIX + i);
+ cache.invoke(0L, EntryUpdater.INSTANCE);
+ }
+ }
+ catch (Throwable ex) {
+ OutOfMemoryException e = findOutOfMemoryException(ex);
+ if (e != null)
+ setOutOfMemoryEx(e);
+ else
+ ignite.log().error("CacheUpdater failed", ex);
+ }
+ }
+
+ /** */
+ private OutOfMemoryException findOutOfMemoryException(Throwable ex) {
+ if (ex instanceof OutOfMemoryException)
+ return (OutOfMemoryException)ex;
+ for (Throwable s : ex.getSuppressed()) {
+ OutOfMemoryException e = findOutOfMemoryException(s);
+ if (e != null)
+ return e;
+ }
+ if (ex.getCause() != null) {
+ OutOfMemoryException e = findOutOfMemoryException(ex.getCause());
+ if (e != null)
+ return e;
+ }
+ return null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/869089db/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index 1cf2c14..efa3225 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -22,6 +22,7 @@ import junit.framework.TestSuite;
import org.apache.ignite.GridCacheAffinityBackupsSelfTest;
import org.apache.ignite.IgniteCacheAffinitySelfTest;
import org.apache.ignite.cache.IgniteWarmupClosureSelfTest;
+import org.apache.ignite.cache.LargeEntryUpdateTest;
import org.apache.ignite.cache.affinity.AffinityClientNodeSelfTest;
import org.apache.ignite.cache.affinity.AffinityHistoryCleanupTest;
import org.apache.ignite.cache.affinity.fair.FairAffinityDynamicCacheSelfTest;
@@ -229,6 +230,8 @@ public class IgniteCacheTestSuite extends TestSuite {
// Warmup closure tests.
suite.addTestSuite(IgniteWarmupClosureSelfTest.class);
+ suite.addTestSuite(LargeEntryUpdateTest.class);
+
// Swap tests.
suite.addTestSuite(GridCacheSwapPreloadSelfTest.class);
[23/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0ee11654
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0ee11654
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0ee11654
Branch: refs/heads/ignite-4712
Commit: 0ee11654cf0580bf3401d69c5c4e3030000dd477
Parents: c762434
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Mon Feb 20 18:25:07 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:54 2017 +0300
----------------------------------------------------------------------
.../database/IgniteDbAbstractTest.java | 14 +++++++--
.../IgniteDbMemoryLeakNonTransactionalTest.java | 31 ++++++++++++++++++++
2 files changed, 43 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0ee11654/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
index 9297cec..5f4d5e0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
@@ -56,7 +56,7 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
protected abstract boolean indexingEnabled();
/** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ @SuppressWarnings("unchecked") @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(gridName);
MemoryConfiguration dbCfg = new MemoryConfiguration();
@@ -112,11 +112,21 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
ccfg4.setRebalanceMode(CacheRebalanceMode.SYNC);
ccfg4.setAffinity(new RendezvousAffinityFunction(false, 32));
+ CacheConfiguration ccfg5 = new CacheConfiguration("atomic");
+
+ if (indexingEnabled())
+ ccfg5.setIndexedTypes(DbKey.class, DbValue.class);
+
+ ccfg5.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+ ccfg5.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg5.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg5.setAffinity(new RendezvousAffinityFunction(false, 32));
+
final AffinityFunction aff = new RendezvousAffinityFunction(1, null);
ccfg4.setAffinity(aff);
- cfg.setCacheConfiguration(ccfg, ccfg2, ccfg3, ccfg4);
+ cfg.setCacheConfiguration(ccfg, ccfg2, ccfg3, ccfg4, ccfg5);
TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
http://git-wip-us.apache.org/repos/asf/ignite/blob/0ee11654/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakNonTransactionalTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakNonTransactionalTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakNonTransactionalTest.java
new file mode 100644
index 0000000..2a6293d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakNonTransactionalTest.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.internal.IgniteEx;
+
+/**
+ *
+ */
+public class IgniteDbMemoryLeakNonTransactionalTest extends IgniteDbMemoryLeakTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
+ return ig.cache("atomic");
+ }
+}
[02/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/95faf450
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/95faf450
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/95faf450
Branch: refs/heads/ignite-4712
Commit: 95faf450cbbf6be4887965209bec3ae0244bcdd9
Parents: 4376f82
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Mon Feb 20 18:11:34 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Tue Feb 21 12:44:29 2017 +0300
----------------------------------------------------------------------
.../IgniteDbMemoryLeakAbstractTest.java | 24 ++++++++------------
.../IgniteDbMemoryLeakLargeObjectsTest.java | 8 -------
.../IgniteDbMemoryLeakLargePagesTest.java | 9 --------
.../database/IgniteDbMemoryLeakTest.java | 8 -------
.../IgniteDbMemoryLeakWithExpirationTest.java | 1 -
.../database/IgniteDbMemoryLeakIndexedTest.java | 9 --------
6 files changed, 10 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/95faf450/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index d939311..bef1320 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -18,13 +18,13 @@
package org.apache.ignite.internal.processors.database;
import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.processors.cache.database.DataStructure;
-import org.apache.ignite.internal.util.GridRandom;
import org.jetbrains.annotations.NotNull;
/**
@@ -33,6 +33,10 @@ import org.jetbrains.annotations.NotNull;
public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest {
/** */
@SuppressWarnings("WeakerAccess") protected static final int CONCURRENCY_LEVEL = 8;
+
+ /** */
+ private static final int MIN_PAGE_CACHE_SIZE = 1048576 * CONCURRENCY_LEVEL;
+
/** */
private volatile Exception ex = null;
@@ -51,9 +55,6 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/** */
private long probeCnt = 0;
- /** */
- private static final ThreadLocal<Random> THREAD_LOCAL_RANDOM = new ThreadLocal<>();
-
@Override protected void beforeTest() throws Exception {
super.beforeTest();
DataStructure.rnd = null;
@@ -71,6 +72,9 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/** {@inheritDoc} */
@Override protected void configure(MemoryConfiguration mCfg) {
mCfg.setConcurrencyLevel(CONCURRENCY_LEVEL);
+
+ long size = 1024 * pagesMax() * (isLargePage() ? 16 : 1);
+ mCfg.setPageCacheSize(Math.max(size, MIN_PAGE_CACHE_SIZE));
}
/**
@@ -81,7 +85,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
}
/**
- * @return Warm up duration.
+ * @return Warm up duration in seconds.
*/
@SuppressWarnings("WeakerAccess") protected int warmUp() {
return 300;
@@ -144,21 +148,13 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
* @return Random.
*/
@NotNull protected static Random getRandom() {
- Random rnd = THREAD_LOCAL_RANDOM.get();
-
- if (rnd == null) {
- rnd = new GridRandom();
- THREAD_LOCAL_RANDOM.set(rnd);
- }
-
- return rnd;
+ return ThreadLocalRandom.current();
}
/**
* @throws Exception If failed.
*/
public void testMemoryLeak() throws Exception {
-
final IgniteEx ignite = grid(0);
final IgniteCache<Object, Object> cache = cache(ignite);
http://git-wip-us.apache.org/repos/asf/ignite/blob/95faf450/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
index 0fa9096..3aab102 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
@@ -18,7 +18,6 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
/**
@@ -36,13 +35,6 @@ public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstra
}
/** {@inheritDoc} */
- @Override protected void configure(MemoryConfiguration mCfg) {
- super.configure(mCfg);
-
- mCfg.setPageCacheSize(35840000); // The space for 35000 pages
- }
-
- /** {@inheritDoc} */
@Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
return ig.cache("large");
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/95faf450/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
index 9dd9fdc..540681d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
@@ -17,20 +17,11 @@
package org.apache.ignite.internal.processors.database;
-import org.apache.ignite.configuration.MemoryConfiguration;
-
/**
*
*/
public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakTest {
/** {@inheritDoc} */
- @Override protected void configure(MemoryConfiguration mCfg) {
- super.configure(mCfg);
-
- mCfg.setPageCacheSize(67108864); // The space for 4000 large pages
- }
-
- /** {@inheritDoc} */
@Override protected boolean isLargePage() {
return true;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/95faf450/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
index 94e419d..30ff18f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
@@ -18,7 +18,6 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
/**
@@ -26,13 +25,6 @@ import org.apache.ignite.internal.IgniteEx;
*/
public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
/** {@inheritDoc} */
- @Override protected void configure(MemoryConfiguration mCfg) {
- super.configure(mCfg);
-
- mCfg.setPageCacheSize(20971520); // The space for 20000 pages
- }
-
- /** {@inheritDoc} */
@Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
return ig.cache("non-primitive");
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/95faf450/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
index c1e279b..212daa1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
@@ -18,7 +18,6 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
import javax.cache.expiry.CreatedExpiryPolicy;
http://git-wip-us.apache.org/repos/asf/ignite/blob/95faf450/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
index c917bc1..f6a06c9 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
@@ -17,20 +17,11 @@
package org.apache.ignite.internal.processors.database;
-import org.apache.ignite.configuration.MemoryConfiguration;
-
/**
*
*/
public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakTest {
/** {@inheritDoc} */
- @Override protected void configure(MemoryConfiguration mCfg) {
- super.configure(mCfg);
-
- mCfg.setPageCacheSize(25165824); // The space for 24000 pages
- }
-
- /** {@inheritDoc} */
@Override protected boolean indexingEnabled() {
return true;
}
[18/43] ignite git commit: IGNITE-4694 Add tests to check there are
no memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4694 Add tests to check there are no memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4c3ccb33
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4c3ccb33
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4c3ccb33
Branch: refs/heads/ignite-4712
Commit: 4c3ccb33de73dfa25a0d76493340d27c9ff781f7
Parents: d89e0e6
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Wed Feb 22 13:18:18 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Feb 22 13:18:18 2017 +0300
----------------------------------------------------------------------
.../IgniteDbMemoryLeakAbstractTest.java | 22 +++++++++-----------
1 file changed, 10 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4c3ccb33/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index aa32495..04ac5ea 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -75,7 +75,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
@Override protected void configure(MemoryConfiguration mCfg) {
mCfg.setConcurrencyLevel(CONCURRENCY_LEVEL);
- long size = 1024 * pagesMax() * (isLargePage() ? 16 : 1);
+ long size = (1024 * (isLargePage() ? 16 : 1) + 24) * pagesMax();
mCfg.setPageCacheSize(Math.max(size, MIN_PAGE_CACHE_SIZE));
}
@@ -92,7 +92,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
*/
@SuppressWarnings("WeakerAccess")
protected int warmUp() {
- return 300;
+ return 450;
}
/** {@inheritDoc} */
@@ -172,6 +172,8 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
final IgniteEx ignite = grid(0);
final IgniteCache<Object, Object> cache = cache(ignite);
+
+
Runnable target = new Runnable() {
@Override public void run() {
while (ex == null && System.nanoTime() < endTime) {
@@ -196,11 +198,15 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
threads[i].start();
}
- Thread.sleep(TimeUnit.NANOSECONDS.toMillis(warmUpEndTime - System.nanoTime()));
+ while (ex == null && System.nanoTime() < warmUpEndTime)
+ Thread.sleep(100);
+
+ if (ex != null)
+ throw ex;
info("Warming up is ended.");
- while (System.nanoTime() < endTime) {
+ while (ex == null && System.nanoTime() < endTime) {
try {
check(ignite);
}
@@ -213,9 +219,6 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
Thread.sleep(TimeUnit.SECONDS.toMillis(5));
}
- for (Thread thread : threads)
- thread.join();
-
if (ex != null)
throw ex;
}
@@ -228,11 +231,6 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
*/
protected void check(IgniteEx ig) throws Exception {
long pagesActual = ig.context().cache().context().database().pageMemory().loadedPages();
- long pagesMax = pagesMax();
-
- assertTrue(
- "Maximal allowed pages number is exceeded [allowed=" + pagesMax + ", actual= " + pagesActual + "]",
- pagesActual <= pagesMax);
if (loadedPages > 0) {
delta += pagesActual - loadedPages;
[34/43] ignite git commit: IGNITE-4694 Add tests to check there are
no memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4694 Add tests to check there are no memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b9fcc144
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b9fcc144
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b9fcc144
Branch: refs/heads/ignite-4712
Commit: b9fcc1446ca1296d6f0d751947b4e91ab84cb84d
Parents: 1678e1b
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Tue Feb 21 11:49:35 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:55 2017 +0300
----------------------------------------------------------------------
.../IgniteDbMemoryLeakAbstractTest.java | 19 ++++--
.../IgniteDbMemoryLeakLargeObjectsTest.java | 8 +--
.../database/IgniteDbMemoryLeakTest.java | 4 +-
.../IgniteDbMemoryLeakWithExpirationTest.java | 1 +
.../testsuites/IgniteDbMemoryLeakTestSuite.java | 46 +++++++++++++
.../IgniteDbMemoryLeakSqlQueryTest.java | 71 ++++++++++++++++++++
...IgniteDbMemoryLeakWithIndexingTestSuite.java | 40 +++++++++++
7 files changed, 177 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b9fcc144/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index ccb1fa0..93306d9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -17,7 +17,6 @@
package org.apache.ignite.internal.processors.database;
-import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import org.apache.ignite.IgniteCache;
@@ -25,7 +24,6 @@ import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.processors.cache.database.DataStructure;
-import org.jetbrains.annotations.NotNull;
/**
* Base class for memory leaks tests.
@@ -55,6 +53,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/** */
private long probeCnt = 0;
+ /** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
super.beforeTest();
DataStructure.rnd = null;
@@ -130,7 +129,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
Object key = key();
Object value = value(key);
- switch (getRandom().nextInt(3)) {
+ switch (nextInt(3)) {
case 0:
cache.getAndPut(key, value);
@@ -145,10 +144,18 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
}
/**
- * @return Random.
+ * @param bound Upper bound (exclusive). Must be positive.
+ * @return Random int value.
+ */
+ protected static int nextInt(int bound) {
+ return ThreadLocalRandom.current().nextInt(bound);
+ }
+
+ /**
+ * @return Random int value.
*/
- @NotNull protected static Random getRandom() {
- return ThreadLocalRandom.current();
+ protected static int nextInt() {
+ return ThreadLocalRandom.current().nextInt();
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/b9fcc144/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
index 3aab102..077a1e1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
@@ -31,7 +31,7 @@ public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstra
ARRAY = new int[1024];
for (int i = 0; i < ARRAY.length; i++)
- ARRAY[i] = getRandom().nextInt();
+ ARRAY[i] = nextInt();
}
/** {@inheritDoc} */
@@ -41,16 +41,16 @@ public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstra
/** {@inheritDoc} */
@Override protected Object key() {
- return new LargeDbKey(getRandom().nextInt(10_000), 1024);
+ return new LargeDbKey(nextInt(10_000), 1024);
}
/** {@inheritDoc} */
@Override protected Object value(Object key) {
- return new LargeDbValue("test-value-1-" + getRandom().nextInt(200), "test-value-2-" + getRandom().nextInt(200), ARRAY);
+ return new LargeDbValue("test-value-1-" + nextInt(200), "test-value-2-" + nextInt(200), ARRAY);
}
/** {@inheritDoc} */
@Override protected long pagesMax() {
- return 35000;
+ return 35_000;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b9fcc144/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
index 30ff18f..b8ac8f0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
@@ -31,12 +31,12 @@ public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
/** {@inheritDoc} */
@Override protected Object key() {
- return new DbKey(getRandom().nextInt(200_000));
+ return new DbKey(nextInt(200_000));
}
/** {@inheritDoc} */
@Override protected Object value(Object key) {
- return new DbValue(((DbKey)key).val, "test-value-" + getRandom().nextInt(200), getRandom().nextInt(500));
+ return new DbValue(((DbKey)key).val, "test-value-" + nextInt(200), nextInt(500));
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/b9fcc144/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
index 212daa1..19502b0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
@@ -38,6 +38,7 @@ public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakTest
return ig.cache("non-primitive").withExpiryPolicy(EXPIRY);
}
+ /** {@inheritDoc} */
@Override protected long pagesMax() {
return 7000;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b9fcc144/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakTestSuite.java
new file mode 100644
index 0000000..75b31b0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakTestSuite.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.database.IgniteDbMemoryLeakLargeObjectsTest;
+import org.apache.ignite.internal.processors.database.IgniteDbMemoryLeakLargePagesTest;
+import org.apache.ignite.internal.processors.database.IgniteDbMemoryLeakNonTransactionalTest;
+import org.apache.ignite.internal.processors.database.IgniteDbMemoryLeakTest;
+import org.apache.ignite.internal.processors.database.IgniteDbMemoryLeakWithExpirationTest;
+
+/**
+ * Page memory leaks tests.
+ */
+public class IgniteDbMemoryLeakTestSuite extends TestSuite {
+ /**
+ * @return Test suite.
+ * @throws Exception Thrown in case of the failure.
+ */
+ public static TestSuite suite() throws Exception {
+ TestSuite suite = new TestSuite("Ignite Db Memory Leaks Test Suite");
+
+ suite.addTestSuite(IgniteDbMemoryLeakTest.class);
+ suite.addTestSuite(IgniteDbMemoryLeakWithExpirationTest.class);
+ suite.addTestSuite(IgniteDbMemoryLeakLargePagesTest.class);
+ suite.addTestSuite(IgniteDbMemoryLeakLargeObjectsTest.class);
+ suite.addTestSuite(IgniteDbMemoryLeakNonTransactionalTest.class);
+
+ return suite;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b9fcc144/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakSqlQueryTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakSqlQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakSqlQueryTest.java
new file mode 100644
index 0000000..b65cac0
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakSqlQueryTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ *
+ */
+public class IgniteDbMemoryLeakSqlQueryTest extends IgniteDbMemoryLeakTest {
+ /** {@inheritDoc} */
+ @Override protected boolean indexingEnabled() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long pagesMax() {
+ return 24_000;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void operation(IgniteCache<Object, Object> cache) {
+ Object key = key();
+ Object value = value(key);
+
+ switch (nextInt(4)) {
+ case 0:
+ cache.getAndPut(key, value);
+
+ break;
+ case 1:
+ cache.get(key);
+
+ break;
+ case 2:
+ cache.getAndRemove(key);
+
+ break;
+ case 3:
+ cache.query(sqlQuery(cache)).getAll();
+ }
+ }
+
+ /**
+ * @param cache IgniteCache.
+ * @return SqlFieldsQuery.
+ */
+ @NotNull private SqlFieldsQuery sqlQuery(IgniteCache<Object, Object> cache) {
+ String query = String.format("select _key from \"%s\".DbValue where iVal=?", cache.getName());
+ SqlFieldsQuery sqlQuery = new SqlFieldsQuery(query);
+ sqlQuery.setArgs(nextInt(200_000));
+ return sqlQuery;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b9fcc144/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakWithIndexingTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakWithIndexingTestSuite.java
new file mode 100644
index 0000000..36cd101
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteDbMemoryLeakWithIndexingTestSuite.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.database.IgniteDbMemoryLeakIndexedTest;
+import org.apache.ignite.internal.processors.database.IgniteDbMemoryLeakSqlQueryTest;
+
+/**
+ * Page memory leaks tests using indexing.
+ */
+public class IgniteDbMemoryLeakWithIndexingTestSuite extends TestSuite {
+ /**
+ * @return Test suite.
+ * @throws Exception Thrown in case of the failure.
+ */
+ public static TestSuite suite() throws Exception {
+ TestSuite suite = new TestSuite("Ignite Db Memory Leaks With Indexing Test Suite");
+
+ suite.addTestSuite(IgniteDbMemoryLeakSqlQueryTest.class);
+ suite.addTestSuite(IgniteDbMemoryLeakIndexedTest.class);
+
+ return suite;
+ }
+}
[22/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c7624343
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c7624343
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c7624343
Branch: refs/heads/ignite-4712
Commit: c7624343d74b37cbcf5813998bd14d2fc2f3560f
Parents: 6e56fa2
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Mon Feb 20 18:11:34 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:54 2017 +0300
----------------------------------------------------------------------
.../IgniteDbMemoryLeakAbstractTest.java | 24 ++++++++------------
.../IgniteDbMemoryLeakLargeObjectsTest.java | 8 -------
.../IgniteDbMemoryLeakLargePagesTest.java | 9 --------
.../database/IgniteDbMemoryLeakTest.java | 8 -------
.../IgniteDbMemoryLeakWithExpirationTest.java | 1 -
.../database/IgniteDbMemoryLeakIndexedTest.java | 9 --------
6 files changed, 10 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7624343/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index d939311..bef1320 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -18,13 +18,13 @@
package org.apache.ignite.internal.processors.database;
import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.processors.cache.database.DataStructure;
-import org.apache.ignite.internal.util.GridRandom;
import org.jetbrains.annotations.NotNull;
/**
@@ -33,6 +33,10 @@ import org.jetbrains.annotations.NotNull;
public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest {
/** */
@SuppressWarnings("WeakerAccess") protected static final int CONCURRENCY_LEVEL = 8;
+
+ /** */
+ private static final int MIN_PAGE_CACHE_SIZE = 1048576 * CONCURRENCY_LEVEL;
+
/** */
private volatile Exception ex = null;
@@ -51,9 +55,6 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/** */
private long probeCnt = 0;
- /** */
- private static final ThreadLocal<Random> THREAD_LOCAL_RANDOM = new ThreadLocal<>();
-
@Override protected void beforeTest() throws Exception {
super.beforeTest();
DataStructure.rnd = null;
@@ -71,6 +72,9 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/** {@inheritDoc} */
@Override protected void configure(MemoryConfiguration mCfg) {
mCfg.setConcurrencyLevel(CONCURRENCY_LEVEL);
+
+ long size = 1024 * pagesMax() * (isLargePage() ? 16 : 1);
+ mCfg.setPageCacheSize(Math.max(size, MIN_PAGE_CACHE_SIZE));
}
/**
@@ -81,7 +85,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
}
/**
- * @return Warm up duration.
+ * @return Warm up duration in seconds.
*/
@SuppressWarnings("WeakerAccess") protected int warmUp() {
return 300;
@@ -144,21 +148,13 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
* @return Random.
*/
@NotNull protected static Random getRandom() {
- Random rnd = THREAD_LOCAL_RANDOM.get();
-
- if (rnd == null) {
- rnd = new GridRandom();
- THREAD_LOCAL_RANDOM.set(rnd);
- }
-
- return rnd;
+ return ThreadLocalRandom.current();
}
/**
* @throws Exception If failed.
*/
public void testMemoryLeak() throws Exception {
-
final IgniteEx ignite = grid(0);
final IgniteCache<Object, Object> cache = cache(ignite);
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7624343/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
index 0fa9096..3aab102 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
@@ -18,7 +18,6 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
/**
@@ -36,13 +35,6 @@ public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstra
}
/** {@inheritDoc} */
- @Override protected void configure(MemoryConfiguration mCfg) {
- super.configure(mCfg);
-
- mCfg.setPageCacheSize(35840000); // The space for 35000 pages
- }
-
- /** {@inheritDoc} */
@Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
return ig.cache("large");
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7624343/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
index 9dd9fdc..540681d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
@@ -17,20 +17,11 @@
package org.apache.ignite.internal.processors.database;
-import org.apache.ignite.configuration.MemoryConfiguration;
-
/**
*
*/
public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakTest {
/** {@inheritDoc} */
- @Override protected void configure(MemoryConfiguration mCfg) {
- super.configure(mCfg);
-
- mCfg.setPageCacheSize(67108864); // The space for 4000 large pages
- }
-
- /** {@inheritDoc} */
@Override protected boolean isLargePage() {
return true;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7624343/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
index 94e419d..30ff18f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
@@ -18,7 +18,6 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
/**
@@ -26,13 +25,6 @@ import org.apache.ignite.internal.IgniteEx;
*/
public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
/** {@inheritDoc} */
- @Override protected void configure(MemoryConfiguration mCfg) {
- super.configure(mCfg);
-
- mCfg.setPageCacheSize(20971520); // The space for 20000 pages
- }
-
- /** {@inheritDoc} */
@Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
return ig.cache("non-primitive");
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7624343/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
index c1e279b..212daa1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
@@ -18,7 +18,6 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
import javax.cache.expiry.CreatedExpiryPolicy;
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7624343/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
index c917bc1..f6a06c9 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
@@ -17,20 +17,11 @@
package org.apache.ignite.internal.processors.database;
-import org.apache.ignite.configuration.MemoryConfiguration;
-
/**
*
*/
public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakTest {
/** {@inheritDoc} */
- @Override protected void configure(MemoryConfiguration mCfg) {
- super.configure(mCfg);
-
- mCfg.setPageCacheSize(25165824); // The space for 24000 pages
- }
-
- /** {@inheritDoc} */
@Override protected boolean indexingEnabled() {
return true;
}
[07/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/be241d97
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/be241d97
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/be241d97
Branch: refs/heads/ignite-4712
Commit: be241d9789392f13c4951e9588c85357cd79192f
Parents: fa395ad
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Mon Feb 20 16:19:00 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Tue Feb 21 12:44:29 2017 +0300
----------------------------------------------------------------------
.../cache/database/freelist/PagesList.java | 44 +++++++++++++-------
1 file changed, 28 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/be241d97/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
index c03f432..1f537b6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
@@ -22,6 +22,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.internal.pagemem.Page;
@@ -76,7 +77,7 @@ public abstract class PagesList extends DataStructure {
Math.min(8, Runtime.getRuntime().availableProcessors() * 2));
/** */
- protected final LongAdder8[] bucketsSize;
+ protected final AtomicLong[] bucketsSize;
/** Page ID to store list metadata. */
private final long metaPageId;
@@ -114,7 +115,7 @@ public abstract class PagesList extends DataStructure {
return TRUE;
}
- };
+ }
/**
* @param cacheId Cache ID.
@@ -139,10 +140,10 @@ public abstract class PagesList extends DataStructure {
this.buckets = buckets;
this.metaPageId = metaPageId;
- bucketsSize = new LongAdder8[buckets];
+ bucketsSize = new AtomicLong[buckets];
for (int i = 0; i < buckets; i++)
- bucketsSize[i] = new LongAdder8();
+ bucketsSize[i] = new AtomicLong();
}
/**
@@ -421,6 +422,7 @@ public abstract class PagesList extends DataStructure {
else {
// It is safe to assign new tail since we do it only when write lock on tail is held.
tails[idx].tailId = newTailId;
+ tails[idx].empty = false;
return true;
}
@@ -597,7 +599,7 @@ public abstract class PagesList extends DataStructure {
if (idx == -1)
handlePageFull(pageId, page, pageAddr, io, dataPage, dataPageAddr, bucket);
else {
- bucketsSize[bucket].increment();
+ bucketsSize[bucket].incrementAndGet();
if (isWalDeltaRecordNeeded(wal, page))
wal.log(new PagesListAddPageRecord(cacheId, pageId, dataPageId));
@@ -691,7 +693,7 @@ public abstract class PagesList extends DataStructure {
assert idx != -1;
- bucketsSize[bucket].increment();
+ bucketsSize[bucket].incrementAndGet();
dataIO.setFreeListPageId(dataPageAddr, nextId);
@@ -781,7 +783,7 @@ public abstract class PagesList extends DataStructure {
}
}
else {
- bucketsSize[bucket].increment();
+ bucketsSize[bucket].incrementAndGet();
// TODO: use single WAL record for bag?
if (isWalDeltaRecordNeeded(wal, page))
@@ -810,7 +812,7 @@ public abstract class PagesList extends DataStructure {
private Stripe getPageForTake(int bucket) {
Stripe[] tails = getBucket(bucket);
- if (tails == null)
+ if (tails == null || bucketsSize[bucket].get() == 0)
return null;
int len = tails.length;
@@ -893,6 +895,16 @@ public abstract class PagesList extends DataStructure {
continue;
}
+ if(!isReuseBucket(bucket) && stripe.empty) {
+ // Another thread took the last page
+ writeUnlock(tail, tailPageAddr, false);
+
+ if(bucketsSize[bucket].get() > 0)
+ continue;
+ else
+ return 0L;
+ }
+
assert PageIO.getPageId(tailPageAddr) == tailId : "tailId = " + tailId + ", tailPageId = " + PageIO.getPageId(tailPageAddr);
assert PageIO.getType(tailPageAddr) == PageIO.T_PAGE_LIST_NODE;
@@ -909,7 +921,7 @@ public abstract class PagesList extends DataStructure {
long pageId = io.takeAnyPage(tailPageAddr);
if (pageId != 0L) {
- bucketsSize[bucket].decrement();
+ bucketsSize[bucket].decrementAndGet();
if (isWalDeltaRecordNeeded(wal, tail))
wal.log(new PagesListRemovePageRecord(cacheId, tailId, pageId));
@@ -918,9 +930,13 @@ public abstract class PagesList extends DataStructure {
ret = pageId;
+ boolean empty = io.isEmpty(tailPageAddr);
+
+ stripe.empty = empty;
+
// If we got an empty page in non-reuse bucket, move it back to reuse list
// to prevent empty page leak to data pages.
- if (io.isEmpty(tailPageAddr) && !isReuseBucket(bucket)) {
+ if (empty && !isReuseBucket(bucket)) {
long prevId = io.getPreviousId(tailPageAddr);
if (prevId != 0L) {
@@ -969,12 +985,8 @@ public abstract class PagesList extends DataStructure {
ret = tailId;
}
- else {
+ else
stripe.empty = true;
-
- if (bucketsSize[bucket].sum() > 0)
- continue;
- }
}
// If we do not have a previous page (we are at head), then we still can return
@@ -1034,7 +1046,7 @@ public abstract class PagesList extends DataStructure {
if (!rmvd)
return false;
- bucketsSize[bucket].decrement();
+ bucketsSize[bucket].decrementAndGet();
if (isWalDeltaRecordNeeded(wal, page))
wal.log(new PagesListRemovePageRecord(cacheId, pageId, dataPageId));
[14/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/fd19998e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/fd19998e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/fd19998e
Branch: refs/heads/ignite-4712
Commit: fd19998e042a43ef1fadd526d7a9693abe1c78d1
Parents: d376ea9
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Wed Feb 22 10:59:06 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Feb 22 10:59:06 2017 +0300
----------------------------------------------------------------------
.../cache/database/freelist/PagesList.java | 178 +++++++++----------
1 file changed, 85 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/fd19998e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
index 7c88041..a5fb617 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
@@ -201,26 +201,43 @@ public abstract class PagesList extends DataStructure {
for (int i = 0; i < upd.length; i++) {
long tailId = upd[i];
- // TODO: need get full bucket size.
- try (Page tail = page(tailId)) {
- long tailAddr = readLock(tail);
+ List<Page> locked = new ArrayList<>(2);
+ List<Long> lockedAddrs = new ArrayList<>(2);
- assert tailAddr != 0L;
+ try {
+ long pageId = tailId;
+ int count = 0;
+
+ while (pageId != 0L) {
+ try (Page page = page(pageId)) {
+ long pageAddr = readLock(page);
- try {
- PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(tailAddr);
+ assert pageAddr != 0L;
- int count = io.getCount(tailAddr);
+ locked.add(page);
+ lockedAddrs.add(pageAddr);
- Stripe stripe = new Stripe(tailId);
- stripe.empty = count == 0;
+ PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(pageAddr);
- tails[i] = stripe;
+ count += io.getCount(pageAddr);
+ pageId = io.getPreviousId(pageAddr);
- bucketSize += count;
+ // In reuse bucket the page itself can be used as a free page
+ if(isReuseBucket(bucket) && pageId != 0L)
+ count++;
+ }
}
- finally {
- readUnlock(tail, tailAddr);
+
+ Stripe stripe = new Stripe(tailId);
+ stripe.empty = count == 0;
+
+ tails[i] = stripe;
+
+ bucketSize += count;
+ }
+ finally {
+ for (int j = 0; j < locked.size(); j++) {
+ readUnlock(locked.get(j), lockedAddrs.get(j));
}
}
}
@@ -389,6 +406,7 @@ public abstract class PagesList extends DataStructure {
}
Stripe stripe = new Stripe(pageId);
+ stripe.empty = true;
for (;;) {
Stripe[] old = getBucket(bucket);
@@ -449,7 +467,6 @@ public abstract class PagesList extends DataStructure {
else {
// It is safe to assign new tail since we do it only when write lock on tail is held.
tails[idx].tailId = newTailId;
- tails[idx].empty = false;
return true;
}
@@ -501,41 +518,11 @@ public abstract class PagesList extends DataStructure {
}
/**
- * !!! For tests only, does not provide any correctness guarantees for concurrent access.
- *
* @param bucket Bucket index.
* @return Number of pages stored in this list.
- * @throws IgniteCheckedException If failed.
*/
- protected final long storedPagesCount(int bucket) throws IgniteCheckedException {
- long res = 0;
-
- Stripe[] tails = getBucket(bucket);
-
- if (tails != null) {
- for (Stripe tail : tails) {
- long pageId = tail.tailId;
-
- try (Page page = page(pageId)) {
- long pageAddr = readLock(page); // No correctness guaranties.
-
- try {
- PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(pageAddr);
-
- int cnt = io.getCount(pageAddr);
-
- assert cnt >= 0;
-
- res += cnt;
- }
- finally {
- readUnlock(page, pageAddr);
- }
- }
- }
- }
-
- return res;
+ protected final long storedPagesCount(int bucket) {
+ return bucketsSize[bucket].get();
}
/**
@@ -685,6 +672,8 @@ public abstract class PagesList extends DataStructure {
dataPageId,
pageId, 0L));
+ bucketsSize[bucket].incrementAndGet();
+
updateTail(bucket, pageId, dataPageId);
}
else {
@@ -720,13 +709,13 @@ public abstract class PagesList extends DataStructure {
assert idx != -1;
- bucketsSize[bucket].incrementAndGet();
-
dataIO.setFreeListPageId(dataPageAddr, nextId);
if (isWalDeltaRecordNeeded(wal, dataPage))
wal.log(new DataPageSetFreeListPageRecord(cacheId, dataPageId, nextId));
+ bucketsSize[bucket].incrementAndGet();
+
updateTail(bucket, pageId, nextId);
}
finally {
@@ -802,6 +791,9 @@ public abstract class PagesList extends DataStructure {
0L
));
+ if(isReuseBucket(bucket))
+ bucketsSize[bucket].incrementAndGet();
+
// Switch to this new page, which is now a part of our list
// to add the rest of the bag to the new page.
prevPageAddr = nextPageAddr;
@@ -810,11 +802,11 @@ public abstract class PagesList extends DataStructure {
}
}
else {
- bucketsSize[bucket].incrementAndGet();
-
// TODO: use single WAL record for bag?
if (isWalDeltaRecordNeeded(wal, page))
wal.log(new PagesListAddPageRecord(cacheId, prevId, nextId));
+
+ bucketsSize[bucket].incrementAndGet();
}
}
}
@@ -923,13 +915,12 @@ public abstract class PagesList extends DataStructure {
continue;
}
- // TODO: condition !isReuseBucket(bucket) is not correct.
- if (!isReuseBucket(bucket) && stripe.empty) {
+ if (stripe.empty) {
// Another thread took the last page.
writeUnlock(tail, tailPageAddr, false);
if (bucketsSize[bucket].get() > 0) {
- lockAttempt = 0;
+ lockAttempt--; // Ignore current attempt.
continue;
}
@@ -941,13 +932,14 @@ public abstract class PagesList extends DataStructure {
assert PageIO.getType(tailPageAddr) == PageIO.T_PAGE_LIST_NODE;
boolean dirty = false;
- long ret = 0L;
+ long ret;
long recycleId = 0L;
try {
PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(tailPageAddr);
if (io.getNextId(tailPageAddr) != 0)
+ // It is not a tail anymore, retry.
continue;
long pageId = io.takeAnyPage(tailPageAddr);
@@ -962,64 +954,64 @@ public abstract class PagesList extends DataStructure {
ret = pageId;
- boolean empty = io.isEmpty(tailPageAddr);
-
- // TODO: add comment, it seems flag is not set to correct value for reuse bucket.
- stripe.empty = empty;
-
- // If we got an empty page in non-reuse bucket, move it back to reuse list
- // to prevent empty page leak to data pages.
- if (empty && !isReuseBucket(bucket)) {
+ if (io.isEmpty(tailPageAddr)) {
long prevId = io.getPreviousId(tailPageAddr);
- if (prevId != 0L) {
- try (Page prev = page(prevId)) {
- // Lock pages from next to previous.
- Boolean ok = writePage(pageMem, prev, this, cutTail, null, bucket, FALSE);
+ // If we got an empty page in non-reuse bucket, move it back to reuse list
+ // to prevent empty page leak to data pages.
+ if (!isReuseBucket(bucket)) {
+ if (prevId != 0L) {
+ try (Page prev = page(prevId)) {
+ // Lock pages from next to previous.
+ Boolean ok = writePage(pageMem, prev, this, cutTail, null, bucket, FALSE);
- assert ok == TRUE : ok;
- }
+ assert ok == TRUE : ok;
+ }
- recycleId = recyclePage(tailId, tail, tailPageAddr);
+ recycleId = recyclePage(tailId, tail, tailPageAddr);
+ }
+ else
+ stripe.empty = true;
}
+ else
+ stripe.empty = prevId == 0L;
}
}
else {
- // The tail page is empty. We can unlink and return it if we have a previous page.
+ // The tail page is empty, but stripe is not. It might
+ // happen only if we are in reuse bucket and it has
+ // a previous page, so, the current page can be collected
+ assert isReuseBucket(bucket);
+
long prevId = io.getPreviousId(tailPageAddr);
- if (prevId != 0L) {
- // This can only happen if we are in the reuse bucket.
- assert isReuseBucket(bucket);
+ assert prevId != 0L;
- try (Page prev = page(prevId)) {
- // Lock pages from next to previous.
- Boolean ok = writePage(pageMem, prev, this, cutTail, null, bucket, FALSE);
+ try (Page prev = page(prevId)) {
+ // Lock pages from next to previous.
+ Boolean ok = writePage(pageMem, prev, this, cutTail, null, bucket, FALSE);
- assert ok == TRUE : ok;
- }
+ assert ok == TRUE : ok;
+ }
- if (initIoVers != null) {
- tailId = PageIdUtils.changeType(tailId, FLAG_DATA);
+ if (initIoVers != null) {
+ tailId = PageIdUtils.changeType(tailId, FLAG_DATA);
- PageIO initIo = initIoVers.latest();
+ PageIO initIo = initIoVers.latest();
- initIo.initNewPage(tailPageAddr, tailId, pageSize());
+ initIo.initNewPage(tailPageAddr, tailId, pageSize());
- if (isWalDeltaRecordNeeded(wal, tail)) {
- wal.log(new InitNewPageRecord(cacheId, tail.id(), initIo.getType(),
- initIo.getVersion(), tailId));
- }
+ if (isWalDeltaRecordNeeded(wal, tail)) {
+ wal.log(new InitNewPageRecord(cacheId, tail.id(), initIo.getType(),
+ initIo.getVersion(), tailId));
}
- else
- tailId = recyclePage(tailId, tail, tailPageAddr);
-
- dirty = true;
-
- ret = tailId;
}
else
- stripe.empty = true;
+ tailId = recyclePage(tailId, tail, tailPageAddr);
+
+ dirty = true;
+
+ ret = tailId;
}
// If we do not have a previous page (we are at head), then we still can return
[12/43] ignite git commit: IGNITE-4712 Memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4712 Memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/bfb0021b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/bfb0021b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/bfb0021b
Branch: refs/heads/ignite-4712
Commit: bfb0021b9fbda0ca9fde83ba5513ed3eb32f3703
Parents: 95faf45
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Mon Feb 20 18:25:07 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Tue Feb 21 12:44:30 2017 +0300
----------------------------------------------------------------------
.../database/IgniteDbAbstractTest.java | 14 +++++++--
.../IgniteDbMemoryLeakNonTransactionalTest.java | 31 ++++++++++++++++++++
2 files changed, 43 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/bfb0021b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
index 9297cec..5f4d5e0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
@@ -56,7 +56,7 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
protected abstract boolean indexingEnabled();
/** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ @SuppressWarnings("unchecked") @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(gridName);
MemoryConfiguration dbCfg = new MemoryConfiguration();
@@ -112,11 +112,21 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
ccfg4.setRebalanceMode(CacheRebalanceMode.SYNC);
ccfg4.setAffinity(new RendezvousAffinityFunction(false, 32));
+ CacheConfiguration ccfg5 = new CacheConfiguration("atomic");
+
+ if (indexingEnabled())
+ ccfg5.setIndexedTypes(DbKey.class, DbValue.class);
+
+ ccfg5.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+ ccfg5.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg5.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg5.setAffinity(new RendezvousAffinityFunction(false, 32));
+
final AffinityFunction aff = new RendezvousAffinityFunction(1, null);
ccfg4.setAffinity(aff);
- cfg.setCacheConfiguration(ccfg, ccfg2, ccfg3, ccfg4);
+ cfg.setCacheConfiguration(ccfg, ccfg2, ccfg3, ccfg4, ccfg5);
TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
http://git-wip-us.apache.org/repos/asf/ignite/blob/bfb0021b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakNonTransactionalTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakNonTransactionalTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakNonTransactionalTest.java
new file mode 100644
index 0000000..2a6293d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakNonTransactionalTest.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.internal.IgniteEx;
+
+/**
+ *
+ */
+public class IgniteDbMemoryLeakNonTransactionalTest extends IgniteDbMemoryLeakTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
+ return ig.cache("atomic");
+ }
+}
[20/43] ignite git commit: [GG-11997] OutOfMemoryException when
repeatedly updating the same entry larger than (page size / 2)
Posted by sb...@apache.org.
[GG-11997] OutOfMemoryException when repeatedly updating the same entry larger than (page size / 2)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c826a97c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c826a97c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c826a97c
Branch: refs/heads/ignite-4712
Commit: c826a97c9812e3e418cba864b8d5412da5468027
Parents: a2ae747
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Tue Feb 28 14:28:11 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Tue Feb 28 14:28:11 2017 +0300
----------------------------------------------------------------------
.../ignite/cache/LargeEntryUpdateTest.java | 188 +++++++++++++++++++
.../ignite/testsuites/IgniteCacheTestSuite.java | 3 +
2 files changed, 191 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c826a97c/modules/core/src/test/java/org/apache/ignite/cache/LargeEntryUpdateTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/LargeEntryUpdateTest.java b/modules/core/src/test/java/org/apache/ignite/cache/LargeEntryUpdateTest.java
new file mode 100644
index 0000000..6b63296
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/LargeEntryUpdateTest.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.cache;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.internal.mem.OutOfMemoryException;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * LargeEntryUpdateTest
+ *
+ * @author Alexandr Kuramshin <ei...@gmail.com>
+ */
+public class LargeEntryUpdateTest extends GridCommonAbstractTest {
+
+ /** */
+ private static final int THREAD_COUNT = 10;
+
+ /** */
+ private static final int PAGE_SIZE = 1 << 10; // 1 kB
+
+ /** */
+ private static final int PAGE_CACHE_SIZE = 30 << 20; // 30 MB
+
+ /** */
+ private static final String CACHE_PREFIX = "testCache";
+
+ /** */
+ private static final int CACHE_COUNT = 10;
+
+ /** */
+ private static final long WAIT_TIMEOUT = 5 * 60_000L; // 5 min
+
+ /** */
+ private static final long TEST_TIMEOUT = 10 * 60_000L; // 10 min
+
+ /** */
+ private final AtomicBoolean cacheUpdate = new AtomicBoolean();
+
+ /** */
+ private OutOfMemoryException outOfMemoryEx;
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return TEST_TIMEOUT;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+ cfg.setPublicThreadPoolSize(THREAD_COUNT);
+
+ MemoryConfiguration mem = new MemoryConfiguration();
+ mem.setPageSize(PAGE_SIZE);
+ mem.setPageCacheSize(PAGE_CACHE_SIZE);
+ cfg.setMemoryConfiguration(mem);
+
+ CacheConfiguration[] ccfgs = new CacheConfiguration[CACHE_COUNT];
+ for (int i = 0; i < CACHE_COUNT; ++i) {
+ CacheConfiguration ccfg = new CacheConfiguration();
+ ccfg.setName(CACHE_PREFIX + i);
+ ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+ ccfg.setCacheMode(CacheMode.PARTITIONED);
+ ccfgs[i] = ccfg;
+ }
+ cfg.setCacheConfiguration(ccfgs);
+
+ return cfg;
+ }
+
+ /** */
+ public void testEntryUpdate() throws Exception {
+ try (Ignite ignite = startGrid()) {
+ for (int i = 0; i < CACHE_COUNT; ++i) {
+ IgniteCache<Long, byte[]> cache = ignite.cache(CACHE_PREFIX + i);
+ cache.put(0L, new byte[PAGE_SIZE * 2]);
+ }
+ cacheUpdate.set(true);
+ IgniteCompute compute = ignite.compute().withAsync();
+ for (int i = 0; i < THREAD_COUNT; ++i)
+ compute.run(new CacheUpdater());
+ try {
+ long deadline = System.currentTimeMillis() + WAIT_TIMEOUT;
+ while (true)
+ synchronized (this) {
+ if (outOfMemoryEx != null)
+ throw outOfMemoryEx;
+ long timeout = deadline - System.currentTimeMillis();
+ if (timeout <= 0)
+ break;
+ wait(timeout);
+ }
+ }
+ finally {
+ cacheUpdate.set(false);
+ }
+ }
+ }
+
+ /** */
+ private synchronized void setOutOfMemoryEx(OutOfMemoryException e) {
+ outOfMemoryEx = e;
+ notifyAll();
+ }
+
+ /** */
+ public static class EntryUpdater implements CacheEntryProcessor<Long, byte[], Void> {
+
+ /** */
+ public static final EntryUpdater INSTANCE = new EntryUpdater();
+
+ /** {@inheritDoc} */
+ @Override public Void process(MutableEntry<Long, byte[]> entry, Object... arguments)
+ throws EntryProcessorException {
+
+ entry.setValue(new byte[PAGE_SIZE]);
+ return null;
+ }
+ }
+
+ /** */
+ public class CacheUpdater implements IgniteRunnable {
+
+ /** */
+ @IgniteInstanceResource
+ public transient Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ try {
+ while (cacheUpdate.get())
+ for (int i = 0; i < CACHE_COUNT; ++i) {
+ IgniteCache<Long, byte[]> cache = ignite.cache(CACHE_PREFIX + i);
+ cache.invoke(0L, EntryUpdater.INSTANCE);
+ }
+ }
+ catch (Throwable ex) {
+ OutOfMemoryException e = findOutOfMemoryException(ex);
+ if (e != null)
+ setOutOfMemoryEx(e);
+ else
+ ignite.log().error("CacheUpdater failed", ex);
+ }
+ }
+
+ /** */
+ private OutOfMemoryException findOutOfMemoryException(Throwable ex) {
+ if (ex instanceof OutOfMemoryException)
+ return (OutOfMemoryException)ex;
+ for (Throwable s : ex.getSuppressed()) {
+ OutOfMemoryException e = findOutOfMemoryException(s);
+ if (e != null)
+ return e;
+ }
+ if (ex.getCause() != null) {
+ OutOfMemoryException e = findOutOfMemoryException(ex.getCause());
+ if (e != null)
+ return e;
+ }
+ return null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c826a97c/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index 1cf2c14..efa3225 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -22,6 +22,7 @@ import junit.framework.TestSuite;
import org.apache.ignite.GridCacheAffinityBackupsSelfTest;
import org.apache.ignite.IgniteCacheAffinitySelfTest;
import org.apache.ignite.cache.IgniteWarmupClosureSelfTest;
+import org.apache.ignite.cache.LargeEntryUpdateTest;
import org.apache.ignite.cache.affinity.AffinityClientNodeSelfTest;
import org.apache.ignite.cache.affinity.AffinityHistoryCleanupTest;
import org.apache.ignite.cache.affinity.fair.FairAffinityDynamicCacheSelfTest;
@@ -229,6 +230,8 @@ public class IgniteCacheTestSuite extends TestSuite {
// Warmup closure tests.
suite.addTestSuite(IgniteWarmupClosureSelfTest.class);
+ suite.addTestSuite(LargeEntryUpdateTest.class);
+
// Swap tests.
suite.addTestSuite(GridCacheSwapPreloadSelfTest.class);
[04/43] ignite git commit: IGNITE-4694 Add tests to check there are
no memory leaks in PageMemory (pending)
Posted by sb...@apache.org.
IGNITE-4694 Add tests to check there are no memory leaks in PageMemory (pending)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/84c03e0c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/84c03e0c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/84c03e0c
Branch: refs/heads/ignite-4712
Commit: 84c03e0c522abc90b2d91e514138eac08388abd2
Parents: e70d990
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Thu Feb 16 13:41:51 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Tue Feb 21 12:44:29 2017 +0300
----------------------------------------------------------------------
.../IgniteDbMemoryLeakAbstractTest.java | 118 +++++++++++++++----
.../database/IgniteDbMemoryLeakIndexedTest.java | 29 -----
.../IgniteDbMemoryLeakLargeObjectsTest.java | 16 +--
.../IgniteDbMemoryLeakLargePagesTest.java | 13 +-
.../database/IgniteDbMemoryLeakTest.java | 14 ++-
.../IgniteDbMemoryLeakWithExpirationTest.java | 7 +-
.../database/IgniteDbMemoryLeakIndexedTest.java | 42 +++++++
7 files changed, 165 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/84c03e0c/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index bca3af0..819405e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -30,13 +30,37 @@ import org.jetbrains.annotations.NotNull;
* Base class for memory leaks tests.
*/
public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest {
-
+ /** */
+ @SuppressWarnings("WeakerAccess") protected static final int CONCURRENCY_LEVEL = 8;
/** */
private volatile Exception ex = null;
/** */
+ private long warmUpEndTime;
+
+ /** */
+ private long endTime;
+
+ /** */
+ private long loadedPages = 0;
+
+ /** */
+ private long delta = 0;
+
+ /** */
+ private long probeCnt = 0;
+
+ /** */
private static final ThreadLocal<Random> THREAD_LOCAL_RANDOM = new ThreadLocal<>();
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ long startTime = System.nanoTime();
+ warmUpEndTime = startTime + TimeUnit.SECONDS.toNanos(warmUp());
+ endTime = warmUpEndTime + TimeUnit.SECONDS.toNanos(duration());
+ }
+
/** {@inheritDoc} */
@Override protected void configure(IgniteConfiguration cfg) {
cfg.setMetricsLogFrequency(5000);
@@ -44,10 +68,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/** {@inheritDoc} */
@Override protected void configure(MemoryConfiguration mCfg) {
- int concLvl = Runtime.getRuntime().availableProcessors();
-
- mCfg.setConcurrencyLevel(concLvl);
- mCfg.setPageCacheSize(1024 * 1024 * concLvl); //minimal possible value
+ mCfg.setConcurrencyLevel(CONCURRENCY_LEVEL);
}
/**
@@ -57,6 +78,13 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
return 300;
}
+ /**
+ * @return Warm up duration.
+ */
+ @SuppressWarnings("WeakerAccess") protected int warmUp() {
+ return 300;
+ }
+
/** {@inheritDoc} */
@Override protected int gridCount() {
return 1;
@@ -69,14 +97,14 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
/** {@inheritDoc} */
@Override protected long getTestTimeout() {
- return (duration() + 1) * 1000;
+ return (warmUp() + duration() + 1) * 1000; // One extra second to stop all threads
}
/**
* @param ig Ignite instance.
* @return IgniteCache.
*/
- protected abstract IgniteCache<Object,Object> cache(IgniteEx ig);
+ protected abstract IgniteCache<Object, Object> cache(IgniteEx ig);
/**
* @return Cache key to perform an operation.
@@ -84,8 +112,8 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
protected abstract Object key();
/**
- * @return Cache value to perform an operation.
* @param key Cache key to perform an operation.
+ * @return Cache value to perform an operation.
*/
protected abstract Object value(Object key);
@@ -99,8 +127,11 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
switch (getRandom().nextInt(3)) {
case 0:
cache.getAndPut(key, value);
+
+ break;
case 1:
cache.get(key);
+
break;
case 2:
cache.getAndRemove(key);
@@ -113,7 +144,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
@NotNull protected static Random getRandom() {
Random rnd = THREAD_LOCAL_RANDOM.get();
- if(rnd == null){
+ if (rnd == null) {
rnd = new GridRandom();
THREAD_LOCAL_RANDOM.set(rnd);
}
@@ -125,49 +156,96 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
* @throws Exception If failed.
*/
public void testMemoryLeak() throws Exception {
- final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration());
final IgniteEx ignite = grid(0);
final IgniteCache<Object, Object> cache = cache(ignite);
Runnable target = new Runnable() {
@Override public void run() {
- while (ex == null && System.nanoTime() < end) {
+ while (ex == null && System.nanoTime() < endTime) {
try {
operation(cache);
- check(ignite);
}
catch (Exception e) {
ex = e;
-
break;
}
}
}
};
- Thread[] threads = new Thread[Runtime.getRuntime().availableProcessors()];
+ Thread[] threads = new Thread[CONCURRENCY_LEVEL];
+
+ info("Warming up is started.");
for (int i = 0; i < threads.length; i++) {
threads[i] = new Thread(target);
threads[i].start();
}
- for (Thread thread : threads) {
- thread.join();
+ Thread.sleep(TimeUnit.NANOSECONDS.toMillis(warmUpEndTime - System.nanoTime()));
+
+ info("Warming up is ended.");
+
+ while (System.nanoTime() < endTime) {
+ try {
+ check(ignite);
+ }
+ catch (Exception e) {
+ ex = e;
+
+ break;
+ }
+
+ Thread.sleep(TimeUnit.SECONDS.toMillis(5));
}
- if(ex != null){
+ for (Thread thread : threads)
+ thread.join();
+
+ if (ex != null)
throw ex;
- }
}
/**
- * Callback to check the current state
+ * Callback to check the current state.
*
- * @param ig Ignite instance
+ * @param ig Ignite instance.
* @throws Exception If failed.
*/
protected void check(IgniteEx ig) throws Exception {
+ long pagesActual = ig.context().cache().context().database().pageMemory().loadedPages();
+ long pagesMax = pagesMax();
+
+ assertTrue(
+ "Maximal allowed pages number is exceeded. [allowed=" + pagesMax + "; actual= " + pagesActual + "]",
+ pagesActual <= pagesMax);
+
+ if (loadedPages > 0) {
+ delta += pagesActual - loadedPages;
+ int allowedDelta = pagesDelta();
+
+ if(probeCnt++ > 12) { // we need some statistic first. Minimal statistic is taken for a minute.
+ long actualDelta = delta / probeCnt;
+
+ assertTrue(
+ "Average growth pages in the number is more than expected. [allowed=" + allowedDelta + "; actual=" + actualDelta + "]",
+ actualDelta <= allowedDelta);
+ }
+ }
+
+ loadedPages = pagesActual;
+ }
+
+ /**
+ * @return Maximal allowed pages number.
+ */
+ protected abstract long pagesMax();
+
+ /**
+ * @return Expected average number of pages, on which their total number can grow per 5 seconds.
+ */
+ @SuppressWarnings("WeakerAccess") protected int pagesDelta() {
+ return 5;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/84c03e0c/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
deleted file mode 100644
index acc6c2f..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.database;
-
-/**
- *
- */
-public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakTest {
-
- /** {@inheritDoc} */
- @Override protected boolean indexingEnabled() {
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/84c03e0c/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
index 8943743..c1b1c9b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
@@ -36,23 +36,25 @@ public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstra
/** {@inheritDoc} */
@Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
- return ig.cache("non-primitive");
+ return ig.cache("large");
}
/** {@inheritDoc} */
@Override protected Object key() {
- return new DbKey(getRandom().nextInt(200_000));
+ return new LargeDbKey(getRandom().nextInt(10_000), 1024);
}
/** {@inheritDoc} */
@Override protected Object value(Object key) {
- return new DbValue(((DbKey)key).val, "test-value-" + getRandom().nextInt(200), getRandom().nextInt(500));
+ return new LargeDbValue("test-value-1-" + getRandom().nextInt(200), "test-value-2-" + getRandom().nextInt(200), ARRAY);
}
- /** {@inheritDoc} */
- @Override protected void check(IgniteEx ig) {
- long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+ @Override protected int warmUp() {
+ return 600;
+ }
- assertTrue(pages < 20000);
+ /** {@inheritDoc} */
+ @Override protected long pagesMax() {
+ return 2000000;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/84c03e0c/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
index 8e4d0b4..9dd9fdc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
@@ -18,19 +18,16 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.internal.IgniteEx;
/**
*
*/
public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakTest {
-
/** {@inheritDoc} */
@Override protected void configure(MemoryConfiguration mCfg) {
- int concLvl = Runtime.getRuntime().availableProcessors();
- mCfg.setConcurrencyLevel(concLvl);
- mCfg.setPageCacheSize(1024 * 1024 * concLvl * 16);
+ super.configure(mCfg);
+ mCfg.setPageCacheSize(67108864); // The space for 4000 large pages
}
/** {@inheritDoc} */
@@ -39,9 +36,7 @@ public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakTest {
}
/** {@inheritDoc} */
- @Override protected void check(IgniteEx ig) {
- long pages = ig.context().cache().context().database().pageMemory().loadedPages();
-
- assertTrue(pages < 4000);
+ @Override protected long pagesMax() {
+ return 4000;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/84c03e0c/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
index 81d831b..94e419d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
/**
@@ -25,6 +26,13 @@ import org.apache.ignite.internal.IgniteEx;
*/
public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
/** {@inheritDoc} */
+ @Override protected void configure(MemoryConfiguration mCfg) {
+ super.configure(mCfg);
+
+ mCfg.setPageCacheSize(20971520); // The space for 20000 pages
+ }
+
+ /** {@inheritDoc} */
@Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
return ig.cache("non-primitive");
}
@@ -40,9 +48,7 @@ public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
}
/** {@inheritDoc} */
- @Override protected void check(IgniteEx ig) {
- long pages = ig.context().cache().context().database().pageMemory().loadedPages();
-
- assertTrue(pages < 20000);
+ @Override protected long pagesMax() {
+ return 20_000;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/84c03e0c/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
index a31ffb4..c1e279b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
@@ -39,10 +39,7 @@ public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakTest
return ig.cache("non-primitive").withExpiryPolicy(EXPIRY);
}
- /** {@inheritDoc} */
- @Override protected void check(IgniteEx ig) {
- long pages = ig.context().cache().context().database().pageMemory().loadedPages();
-
- assertTrue(pages < 7000);
+ @Override protected long pagesMax() {
+ return 7000;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/84c03e0c/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
new file mode 100644
index 0000000..c917bc1
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.configuration.MemoryConfiguration;
+
+/**
+ *
+ */
+public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakTest {
+ /** {@inheritDoc} */
+ @Override protected void configure(MemoryConfiguration mCfg) {
+ super.configure(mCfg);
+
+ mCfg.setPageCacheSize(25165824); // The space for 24000 pages
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean indexingEnabled() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long pagesMax() {
+ return 24_000;
+ }
+}
[27/43] ignite git commit: IGNITE-4694 Add tests to check there are
no memory leaks in PageMemory
Posted by sb...@apache.org.
IGNITE-4694 Add tests to check there are no memory leaks in PageMemory
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/25c19276
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/25c19276
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/25c19276
Branch: refs/heads/ignite-4712
Commit: 25c192762aae86d60e689e9cfd90f501618ea753
Parents: 88c04bb
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Wed Feb 15 18:04:38 2017 +0300
Committer: Igor Seliverstov <gv...@gmail.com>
Committed: Wed Mar 1 10:44:54 2017 +0300
----------------------------------------------------------------------
.../cache/IgniteCacheOffheapManagerImpl.java | 2 +-
.../database/IgniteDbAbstractTest.java | 6 +
.../IgniteDbMemoryLeakAbstractTest.java | 172 ++++++++++++++-----
.../database/IgniteDbMemoryLeakIndexedTest.java | 65 +------
.../IgniteDbMemoryLeakLargeObjectsTest.java | 64 ++-----
.../IgniteDbMemoryLeakLargePagesTest.java | 67 ++------
.../database/IgniteDbMemoryLeakTest.java | 63 ++-----
.../IgniteDbMemoryLeakWithExpirationTest.java | 58 +------
8 files changed, 182 insertions(+), 315 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/25c19276/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index b863edd..ad3663c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -906,7 +906,7 @@ public class IgniteCacheOffheapManagerImpl extends GridCacheManagerAdapter imple
*/
private boolean canUpdateOldRow(@Nullable CacheDataRow oldRow, DataRow dataRow)
throws IgniteCheckedException {
- if (oldRow == null || indexingEnabled)
+ if (oldRow == null || indexingEnabled || oldRow.expireTime() != dataRow.expireTime())
return false;
if (oldRow.expireTime() != dataRow.expireTime())
http://git-wip-us.apache.org/repos/asf/ignite/blob/25c19276/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
index 3bc7004..9297cec 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
@@ -130,10 +130,16 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
return cfg;
}
+ /**
+ * @param cfg IgniteConfiguration.
+ */
protected void configure(IgniteConfiguration cfg){
//NOP
}
+ /**
+ * @param mCfg MemoryConfiguration.
+ */
protected void configure(MemoryConfiguration mCfg){
//NOP
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/25c19276/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index fc0e715..bca3af0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -17,69 +17,157 @@
package org.apache.ignite.internal.processors.database;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCompute;
-import org.apache.ignite.compute.ComputeTaskFuture;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.lang.IgniteRunnable;
-import org.apache.ignite.resources.IgniteInstanceResource;
-
+import java.util.Random;
import java.util.concurrent.TimeUnit;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.GridRandom;
+import org.jetbrains.annotations.NotNull;
/**
- * TODO: fix javadoc warnings, code style.
+ * Base class for memory leaks tests.
*/
public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest {
- // TODO: take duration from system property.
- /** Test duration in seconds*/
- protected abstract int duration();
-
- @Override
- protected long getTestTimeout() {
- return duration() * 1200;
- }
/** */
- protected abstract void operation(IgniteEx ig);
+ private volatile Exception ex = null;
/** */
- public void testMemoryLeak() throws Exception {
- // TODO: take PageMemory max size is the same as we configured.
+ private static final ThreadLocal<Random> THREAD_LOCAL_RANDOM = new ThreadLocal<>();
- final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration());
+ /** {@inheritDoc} */
+ @Override protected void configure(IgniteConfiguration cfg) {
+ cfg.setMetricsLogFrequency(5000);
+ }
- // TODO: use threads instead of compute or make sure there are enough threads in pool.
- int tasksCnt = Runtime.getRuntime().availableProcessors() * 4;
+ /** {@inheritDoc} */
+ @Override protected void configure(MemoryConfiguration mCfg) {
+ int concLvl = Runtime.getRuntime().availableProcessors();
- IgniteCompute compute = grid(0).compute().withAsync();
+ mCfg.setConcurrencyLevel(concLvl);
+ mCfg.setPageCacheSize(1024 * 1024 * concLvl); //minimal possible value
+ }
- ComputeTaskFuture[] futs = new ComputeTaskFuture[tasksCnt];
+ /**
+ * @return Test duration in seconds.
+ */
+ protected int duration() {
+ return 300;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int gridCount() {
+ return 1;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean indexingEnabled() {
+ return false;
+ }
- for (int i = 0; i < tasksCnt; i++) {
- compute.run(new IgniteRunnable() {
- @IgniteInstanceResource
- private Ignite ig;
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return (duration() + 1) * 1000;
+ }
- @Override
- public void run() {
- int i = 0;
- while (System.nanoTime() < end) {
- operation((IgniteEx) ig);
+ /**
+ * @param ig Ignite instance.
+ * @return IgniteCache.
+ */
+ protected abstract IgniteCache<Object,Object> cache(IgniteEx ig);
+
+ /**
+ * @return Cache key to perform an operation.
+ */
+ protected abstract Object key();
+
+ /**
+ * @return Cache value to perform an operation.
+ * @param key Cache key to perform an operation.
+ */
+ protected abstract Object value(Object key);
+
+ /**
+ * @param cache IgniteCache.
+ */
+ protected void operation(IgniteCache<Object, Object> cache) {
+ Object key = key();
+ Object value = value(key);
+
+ switch (getRandom().nextInt(3)) {
+ case 0:
+ cache.getAndPut(key, value);
+ case 1:
+ cache.get(key);
+ break;
+ case 2:
+ cache.getAndRemove(key);
+ }
+ }
- if(i++ == 100) {
- check((IgniteEx) ig);
- i = 0;
- }
+ /**
+ * @return Random.
+ */
+ @NotNull protected static Random getRandom() {
+ Random rnd = THREAD_LOCAL_RANDOM.get();
+
+ if(rnd == null){
+ rnd = new GridRandom();
+ THREAD_LOCAL_RANDOM.set(rnd);
+ }
+
+ return rnd;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testMemoryLeak() throws Exception {
+ final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration());
+
+ final IgniteEx ignite = grid(0);
+ final IgniteCache<Object, Object> cache = cache(ignite);
+
+ Runnable target = new Runnable() {
+ @Override public void run() {
+ while (ex == null && System.nanoTime() < end) {
+ try {
+ operation(cache);
+ check(ignite);
+ }
+ catch (Exception e) {
+ ex = e;
+
+ break;
}
}
- });
+ }
+ };
+
+ Thread[] threads = new Thread[Runtime.getRuntime().availableProcessors()];
- futs[i] = compute.future();
+ for (int i = 0; i < threads.length; i++) {
+ threads[i] = new Thread(target);
+ threads[i].start();
}
- for (ComputeTaskFuture fut : futs)
- fut.get();
+ for (Thread thread : threads) {
+ thread.join();
+ }
+
+ if(ex != null){
+ throw ex;
+ }
}
- protected void check(IgniteEx ig) {}
+ /**
+ * Callback to check the current state
+ *
+ * @param ig Ignite instance
+ * @throws Exception If failed.
+ */
+ protected void check(IgniteEx ig) throws Exception {
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/25c19276/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
index db77131..acc6c2f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
@@ -17,72 +17,13 @@
package org.apache.ignite.internal.processors.database;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
-
/**
*
*/
-public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakAbstractTest {
-
- @Override
- protected int duration() {
- return 300;
- }
-
- @Override
- protected int gridCount() {
- return 1;
- }
-
- @Override
- protected void configure(IgniteConfiguration cfg) {
- cfg.setMetricsLogFrequency(5000);
- }
-
- @Override
- protected void configure(MemoryConfiguration mCfg) {
- mCfg.setPageCacheSize(1024 * 1024);
- }
+public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakTest {
- @Override
- // TODO: move test to module ignite-indexing.
- protected boolean indexingEnabled() {
+ /** {@inheritDoc} */
+ @Override protected boolean indexingEnabled() {
return true;
}
-
- protected void operation(IgniteEx ig){
- IgniteCache<Object, Object> cache = ig.cache("non-primitive");
- Random rnd = ThreadLocalRandom.current();
-
- for (int i = 0; i < 1000; i++) {
- DbKey key = new DbKey(rnd.nextInt(200_000));
-
- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
-
- // TODO: also execute sql queries.
-
- switch (rnd.nextInt(3)) {
- case 0:
- cache.getAndPut(key, v0);
- case 1:
- cache.get(key);
- break;
- case 2:
- cache.getAndRemove(key);
- }
- }
- }
-
- @Override
- protected void check(IgniteEx ig) {
- long pages = ig.context().cache().context().database().pageMemory().loadedPages();
-
- assertTrue(pages < 19100);
- }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/25c19276/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
index 2a6c8cd..8943743 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
@@ -18,13 +18,8 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
-
/**
*
*/
@@ -35,62 +30,29 @@ public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstra
static {
ARRAY = new int[1024];
- Random rnd = new Random();
-
for (int i = 0; i < ARRAY.length; i++)
- ARRAY[i] = rnd.nextInt();
- }
-
- @Override
- protected int duration() {
- return 300;
- }
-
- @Override
- protected int gridCount() {
- return 1;
+ ARRAY[i] = getRandom().nextInt();
}
- @Override
- protected void configure(IgniteConfiguration cfg) {
- cfg.setMetricsLogFrequency(5000);
+ /** {@inheritDoc} */
+ @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
+ return ig.cache("non-primitive");
}
- @Override
- protected void configure(MemoryConfiguration mCfg) {
- mCfg.setPageCacheSize(60 * 1024 * 1024);
+ /** {@inheritDoc} */
+ @Override protected Object key() {
+ return new DbKey(getRandom().nextInt(200_000));
}
- @Override
- protected boolean indexingEnabled() {
- return false;
- }
-
- protected void operation(IgniteEx ig){
- IgniteCache<Object, Object> cache = ig.cache("large");
- Random rnd = ThreadLocalRandom.current();
-
- for (int i = 0; i < 1000; i++) {
- LargeDbKey key = new LargeDbKey(rnd.nextInt(10_000), 1024);
-
- LargeDbValue v0 = new LargeDbValue("test-value-1-" + rnd.nextInt(200), "test-value-2-" + rnd.nextInt(200), ARRAY);
-
- switch (rnd.nextInt(3)) {
- case 0:
- cache.getAndPut(key, v0);
- case 1:
- cache.get(key);
- break;
- case 2:
- cache.getAndRemove(key);
- }
- }
+ /** {@inheritDoc} */
+ @Override protected Object value(Object key) {
+ return new DbValue(((DbKey)key).val, "test-value-" + getRandom().nextInt(200), getRandom().nextInt(500));
}
- @Override
- protected void check(IgniteEx ig) {
+ /** {@inheritDoc} */
+ @Override protected void check(IgniteEx ig) {
long pages = ig.context().cache().context().database().pageMemory().loadedPages();
- assertTrue(pages < 50000);
+ assertTrue(pages < 20000);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/25c19276/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
index 91c96af..8e4d0b4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
@@ -17,76 +17,31 @@
package org.apache.ignite.internal.processors.database;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
-
/**
*
*/
-public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakAbstractTest {
-
- @Override
- protected int duration() {
- return 300;
- }
+public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakTest {
- @Override
- protected int gridCount() {
- return 1;
- }
+ /** {@inheritDoc} */
+ @Override protected void configure(MemoryConfiguration mCfg) {
+ int concLvl = Runtime.getRuntime().availableProcessors();
+ mCfg.setConcurrencyLevel(concLvl);
+ mCfg.setPageCacheSize(1024 * 1024 * concLvl * 16);
- @Override
- protected void configure(IgniteConfiguration cfg) {
- cfg.setMetricsLogFrequency(5000);
}
- @Override
- protected void configure(MemoryConfiguration mCfg) {
- // TODO: understand why such overhead with large pages.
- mCfg.setPageCacheSize(100 * 1024 * 1024);
- }
-
- @Override
- protected boolean indexingEnabled() {
- return false;
- }
-
- @Override
- protected boolean isLargePage() {
+ /** {@inheritDoc} */
+ @Override protected boolean isLargePage() {
return true;
}
- // TODO: avoid copy/paste.
- protected void operation(IgniteEx ig){
- IgniteCache<Object, Object> cache = ig.cache("non-primitive");
- Random rnd = ThreadLocalRandom.current();
-
- for (int i = 0; i < 1000; i++) {
- DbKey key = new DbKey(rnd.nextInt(200_000));
-
- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
-
- switch (rnd.nextInt(3)) {
- case 0:
- cache.getAndPut(key, v0);
- case 1:
- cache.get(key);
- break;
- case 2:
- cache.getAndRemove(key);
- }
- }
- }
-
- @Override
- protected void check(IgniteEx ig) {
+ /** {@inheritDoc} */
+ @Override protected void check(IgniteEx ig) {
long pages = ig.context().cache().context().database().pageMemory().loadedPages();
- assertTrue(pages < 4600);
+ assertTrue(pages < 4000);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/25c19276/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
index 2b0ce1e..81d831b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
@@ -18,72 +18,31 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
-
/**
*
*/
public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
/** {@inheritDoc} */
- @Override protected int duration() {
- return 300;
- }
-
- @Override
- protected int gridCount() {
- return 1;
- }
-
- @Override
- protected void configure(IgniteConfiguration cfg) {
- cfg.setMetricsLogFrequency(5000);
- }
-
- @Override
- protected void configure(MemoryConfiguration mCfg) {
- mCfg.setPageCacheSize(1024 * 1024);
+ @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
+ return ig.cache("non-primitive");
}
- @Override
- protected boolean indexingEnabled() {
- return false;
+ /** {@inheritDoc} */
+ @Override protected Object key() {
+ return new DbKey(getRandom().nextInt(200_000));
}
- protected void operation(IgniteEx ig){
- IgniteCache<Object, Object> cache = ig.cache("non-primitive");
- Random rnd = ThreadLocalRandom.current();
-
- for (int i = 0; i < 1000; i++) {
- DbKey key = new DbKey(rnd.nextInt(200_000));
-
- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
-
- // TODO: also execute scan query.
-
- switch (rnd.nextInt(3)) {
- case 0:
- cache.getAndPut(key, v0);
- break;
-
- case 1:
- cache.get(key);
- break;
-
- case 2:
- cache.getAndRemove(key);
- }
- }
+ /** {@inheritDoc} */
+ @Override protected Object value(Object key) {
+ return new DbValue(((DbKey)key).val, "test-value-" + getRandom().nextInt(200), getRandom().nextInt(500));
}
- @Override
- protected void check(IgniteEx ig) {
+ /** {@inheritDoc} */
+ @Override protected void check(IgniteEx ig) {
long pages = ig.context().cache().context().database().pageMemory().loadedPages();
- assertTrue(pages < 19100);
+ assertTrue(pages < 20000);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/25c19276/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
index 95fe8c8..a31ffb4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
@@ -18,75 +18,31 @@
package org.apache.ignite.internal.processors.database;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
import javax.cache.expiry.CreatedExpiryPolicy;
import javax.cache.expiry.Duration;
import javax.cache.expiry.ExpiryPolicy;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
/**
*
*/
-public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakAbstractTest {
+public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakTest {
/** */
private static final ExpiryPolicy EXPIRY = new CreatedExpiryPolicy(new Duration(MILLISECONDS, 10L));
- @Override
- protected int duration() {
- return 300;
+ /** {@inheritDoc} */
+ @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
+ return ig.cache("non-primitive").withExpiryPolicy(EXPIRY);
}
- @Override
- protected int gridCount() {
- return 1;
- }
-
- @Override
- protected void configure(IgniteConfiguration cfg) {
- cfg.setMetricsLogFrequency(5000);
- }
-
- @Override
- protected void configure(MemoryConfiguration mCfg) {
- mCfg.setPageCacheSize(1024 * 1024);
- }
-
- @Override
- protected boolean indexingEnabled() {
- return false;
- }
-
- protected void operation(IgniteEx ig) {
- IgniteCache<Object, Object> cache = ig.cache("non-primitive").withExpiryPolicy(EXPIRY);
- Random rnd = ThreadLocalRandom.current();
-
- for (int i = 0; i < 1000; i++) {
- DbKey key = new DbKey(rnd.nextInt(200_000));
-
- DbValue v0 = new DbValue(key.val, "test-value-" + rnd.nextInt(200), rnd.nextInt(500));
-
- switch (rnd.nextInt(3)) {
- case 0:
- cache.getAndPut(key, v0);
- case 1:
- cache.get(key);
- break;
- case 2:
- cache.getAndRemove(key);
- }
- }
- }
-
- @Override
- protected void check(IgniteEx ig) {
+ /** {@inheritDoc} */
+ @Override protected void check(IgniteEx ig) {
long pages = ig.context().cache().context().database().pageMemory().loadedPages();
- assertTrue(pages < 10000);
+ assertTrue(pages < 7000);
}
}