You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2018/05/07 14:57:15 UTC
[1/3] ignite git commit: IGNITE-8421 new implementation of
getChildren method is added tolerating KeeperException.NoNodeException -
Fixes #3939.
Repository: ignite
Updated Branches:
refs/heads/ignite-2.5 db8865a7b -> d83f1ec0a
IGNITE-8421 new implementation of getChildren method is added tolerating KeeperException.NoNodeException - Fixes #3939.
Signed-off-by: dpavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/34c14ed7
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/34c14ed7
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/34c14ed7
Branch: refs/heads/ignite-2.5
Commit: 34c14ed7c924c2155cbfdb412bb9ca9efa942117
Parents: db8865a
Author: Sergey Chugunov <se...@gmail.com>
Authored: Fri May 4 15:26:02 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Mon May 7 17:19:02 2018 +0300
----------------------------------------------------------------------
.../ZkDistributedCollectDataFuture.java | 2 +-
.../discovery/zk/internal/ZookeeperClient.java | 25 ++++++++++++++++++++
2 files changed, 26 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/34c14ed7/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
index 174d698..e9b28e1 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
@@ -150,7 +150,7 @@ class ZkDistributedCollectDataFuture extends GridFutureAdapter<Void> {
try {
client.deleteAll(evtDir,
- client.getChildren(evtDir),
+ client.getChildrenIfPathExists(evtDir),
-1);
}
catch (KeeperException.NoNodeException e) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/34c14ed7/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
index cc525d3..6cc77a5 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
@@ -491,6 +491,31 @@ public class ZookeeperClient implements Watcher {
/**
* @param path Path.
+ * @return Children nodes.
+ * @throws KeeperException.NoNodeException If provided path does not exist.
+ * @throws ZookeeperClientFailedException If connection to zk was lost.
+ * @throws InterruptedException If interrupted.
+ */
+ List<String> getChildrenIfPathExists(String path) throws
+ KeeperException.NoNodeException, InterruptedException, ZookeeperClientFailedException {
+ for (;;) {
+ long connStartTime = this.connStartTime;
+
+ try {
+ return zk.getChildren(path, false);
+ }
+ catch (KeeperException.NoNodeException e) {
+ throw e;
+ }
+ catch (Exception e) {
+ onZookeeperError(connStartTime, e);
+ }
+ }
+ }
+
+
+ /**
+ * @param path Path.
* @throws InterruptedException If interrupted.
* @throws KeeperException In case of error.
* @return {@code True} if given path exists.
[2/3] ignite git commit: IGNITE-8347 Test of Memory leaks on restart
Ignite node with enabled persistence at ThreadLocal. - Fixes #3889.
Posted by ag...@apache.org.
IGNITE-8347 Test of Memory leaks on restart Ignite node with enabled persistence at ThreadLocal. - Fixes #3889.
Signed-off-by: dpavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8401fe69
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8401fe69
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8401fe69
Branch: refs/heads/ignite-2.5
Commit: 8401fe690cb6b028c7f36ec1f6d7dc8fc4f19fc6
Parents: 34c14ed
Author: tledkov-gridgain <tl...@gridgain.com>
Authored: Thu May 3 16:23:40 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Mon May 7 17:22:23 2018 +0300
----------------------------------------------------------------------
.../apache/ignite/internal/util/GridDebug.java | 69 ++++++++++-
.../internal/MemoryLeaksOnRestartNodeTest.java | 118 +++++++++++++++++++
2 files changed, 186 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8401fe69/modules/core/src/main/java/org/apache/ignite/internal/util/GridDebug.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridDebug.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridDebug.java
index a8af0fb..2fa148e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridDebug.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridDebug.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.util;
+import com.sun.management.HotSpotDiagnosticMXBean;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@@ -32,6 +33,7 @@ import java.util.Collection;
import java.util.Date;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicReference;
+import javax.management.MBeanServer;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgnitePredicate;
@@ -67,7 +69,13 @@ public class GridDebug {
/** */
private static boolean allowLog;
- /** */
+ /** This is the name of the HotSpot Diagnostic MBean */
+ private static final String HOTSPOT_BEAN_NAME = "com.sun.management:type=HotSpotDiagnostic";
+
+ /** field to store the hotspot diagnostic MBean */
+ private static volatile HotSpotDiagnosticMXBean hotspotMBean;
+
+ /* */
static {
if (LOGS_PATH != null) {
File log = new File(new File(LOGS_PATH), new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss-").format(new Date()) +
@@ -303,6 +311,65 @@ public class GridDebug {
}
/**
+ * Call this method from your application whenever you
+ * want to dump the heap snapshot into a file.
+ *
+ * @param fileName name of the heap dump file
+ * @param live flag that tells whether to dump
+ * only the live objects
+ */
+ public static void dumpHeap(String fileName, boolean live) {
+ // initialize hotspot diagnostic MBean
+ initHotspotMBean();
+
+ File f = new File(fileName);
+
+ if (f.exists())
+ f.delete();
+
+ try {
+ hotspotMBean.dumpHeap(fileName, live);
+ }
+ catch (RuntimeException re) {
+ throw re;
+ }
+ catch (Exception exp) {
+ throw new RuntimeException(exp);
+ }
+ }
+
+ /**
+ * Initialize the hotspot diagnostic MBean field
+ */
+ private static void initHotspotMBean() {
+ if (hotspotMBean == null) {
+ synchronized (GridDebug.class) {
+ if (hotspotMBean == null)
+ hotspotMBean = getHotspotMBean();
+ }
+ }
+ }
+
+ /**
+ * Gets the hotspot diagnostic MBean from the platform MBean server
+ * @return Diagnostic bean.
+ */
+ private static HotSpotDiagnosticMXBean getHotspotMBean() {
+ try {
+ MBeanServer server = ManagementFactory.getPlatformMBeanServer();
+
+ HotSpotDiagnosticMXBean bean = ManagementFactory.newPlatformMXBeanProxy(server,
+ HOTSPOT_BEAN_NAME, HotSpotDiagnosticMXBean.class);
+
+ return bean;
+ } catch (RuntimeException re) {
+ throw re;
+ } catch (Exception exp) {
+ throw new RuntimeException(exp);
+ }
+ }
+
+ /**
* Debug info queue item.
*/
@SuppressWarnings({"PublicInnerClass", "PublicField"})
http://git-wip-us.apache.org/repos/asf/ignite/blob/8401fe69/modules/core/src/test/java/org/apache/ignite/internal/MemoryLeaksOnRestartNodeTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/MemoryLeaksOnRestartNodeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/MemoryLeaksOnRestartNodeTest.java
new file mode 100644
index 0000000..56ab091
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/MemoryLeaksOnRestartNodeTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.io.File;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.GridDebug;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests leaks on node restart with enabled persistence.
+ */
+public class MemoryLeaksOnRestartNodeTest extends GridCommonAbstractTest {
+ /** Heap dump file name. */
+ private static final String HEAP_DUMP_FILE_NAME = "test.hprof";
+
+ /** Restarts count. */
+ private static final int RESTARTS = 10;
+
+ /** Nodes count. */
+ private static final int NODES = 3;
+
+ /** Allow 5Mb leaks on node restart. */
+ private static final int ALLOW_LEAK_ON_RESTART_IN_MB = 1;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setDataStorageConfiguration(new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration().setName("mem0").setPersistenceEnabled(false))
+ .setDataRegionConfigurations(
+ new DataRegionConfiguration().setName("disk").setPersistenceEnabled(true),
+ new DataRegionConfiguration().setName("mem2").setPersistenceEnabled(false)));
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+
+ super.afterTestsStopped();
+ }
+
+ /**
+ * @throws Exception On failed.
+ */
+ public void test() throws Exception {
+ System.setProperty(IgniteSystemProperties.IGNITE_DELAYED_REPLACED_PAGE_WRITE, "false");
+
+ // Warmup
+ for (int i = 0; i < RESTARTS / 2; ++i) {
+ startGrids(NODES);
+
+ U.sleep(500);
+
+ stopAllGrids();
+ }
+
+ GridDebug.dumpHeap(HEAP_DUMP_FILE_NAME, true);
+
+ File dumpFile = new File(HEAP_DUMP_FILE_NAME);
+
+ final long size0 = dumpFile.length();
+
+ // Restarts
+ for (int i = 0; i < RESTARTS; ++i) {
+ startGrids(NODES);
+
+ U.sleep(500);
+
+ stopAllGrids();
+
+ GridDebug.dumpHeap(HEAP_DUMP_FILE_NAME, true);
+ }
+
+ GridDebug.dumpHeap(HEAP_DUMP_FILE_NAME, true);
+
+ final float leakSize = (float)(dumpFile.length() - size0) / 1024 / 1024 / NODES / RESTARTS;
+
+ assertTrue("Possible leaks detected. The " + leakSize + "M leaks per node restart after " + RESTARTS
+ + " restarts. See the '" + dumpFile.getAbsolutePath() + "'",
+ leakSize < ALLOW_LEAK_ON_RESTART_IN_MB);
+
+ // Remove dump if successful.
+ dumpFile.delete();
+ }
+}
\ No newline at end of file
[3/3] ignite git commit: IGNITE-8347 Memory leaks on restart Ignite
node with enabled persistence at ThreadLocal - Fixes #3891.
Posted by ag...@apache.org.
IGNITE-8347 Memory leaks on restart Ignite node with enabled persistence at ThreadLocal - Fixes #3891.
Signed-off-by: dpavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d83f1ec0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d83f1ec0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d83f1ec0
Branch: refs/heads/ignite-2.5
Commit: d83f1ec0a4d010e57fecc12ffd3d1f8346ded61c
Parents: 8401fe6
Author: dpavlov <dp...@apache.org>
Authored: Thu May 3 18:26:49 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Mon May 7 17:22:41 2018 +0300
----------------------------------------------------------------------
.../persistence/pagemem/DelayedDirtyPageWrite.java | 6 +++---
.../pagemem/DelayedPageReplacementTracker.java | 16 +++++++---------
.../internal/MemoryLeaksOnRestartNodeTest.java | 3 ---
3 files changed, 10 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d83f1ec0/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/DelayedDirtyPageWrite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/DelayedDirtyPageWrite.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/DelayedDirtyPageWrite.java
index 6eec609..b08ddc2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/DelayedDirtyPageWrite.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/DelayedDirtyPageWrite.java
@@ -24,9 +24,9 @@ import org.apache.ignite.internal.util.GridUnsafe;
import org.jetbrains.annotations.Nullable;
/**
- * Not thread safe and stateful class for replacement of one page with write() delay. This allows to write page content
- * without holding segment lock. Page data is copied into temp buffer during {@link #writePage(FullPageId, ByteBuffer,
- * int)} and then sent to real implementation by {@link #finishReplacement()}.
+ * Not thread safe and stateful class for page replacement of one page with write() delay. This allows to write page
+ * content without holding segment lock. Page data is copied into temp buffer during {@link #writePage(FullPageId,
+ * ByteBuffer, int)} and then sent to real implementation by {@link #finishReplacement()}.
*/
public class DelayedDirtyPageWrite implements ReplacedPageWriter {
/** Real flush dirty page implementation. */
http://git-wip-us.apache.org/repos/asf/ignite/blob/d83f1ec0/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/DelayedPageReplacementTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/DelayedPageReplacementTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/DelayedPageReplacementTracker.java
index 9cf5b77..aa1b061 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/DelayedPageReplacementTracker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/DelayedPageReplacementTracker.java
@@ -21,6 +21,8 @@ import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Collection;
import java.util.HashSet;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.ignite.IgniteInterruptedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.internal.pagemem.FullPageId;
@@ -56,15 +58,10 @@ public class DelayedPageReplacementTracker {
/**
* Dirty page write for replacement operations thread local. Because page write {@link DelayedDirtyPageWrite} is
- * stateful and not thread safe, this thread local protects from GC pressure on pages replacement.
+ * stateful and not thread safe, this thread local protects from GC pressure on pages replacement. <br> Map is used
+ * instead of build-in thread local to allow GC to remove delayed writers for alive threads after node stop.
*/
- private final ThreadLocal<DelayedDirtyPageWrite> delayedPageWriteThreadLoc
- = new ThreadLocal<DelayedDirtyPageWrite>() {
- @Override protected DelayedDirtyPageWrite initialValue() {
- return new DelayedDirtyPageWrite(flushDirtyPage, byteBufThreadLoc, pageSize,
- DelayedPageReplacementTracker.this);
- }
- };
+ private final Map<Long, DelayedDirtyPageWrite> delayedPageWriteThreadLocMap = new ConcurrentHashMap<>();
/**
* @param pageSize Page size.
@@ -87,7 +84,8 @@ public class DelayedPageReplacementTracker {
* @return delayed page write implementation, finish method to be called to actually write page.
*/
public DelayedDirtyPageWrite delayedPageWrite() {
- return delayedPageWriteThreadLoc.get();
+ return delayedPageWriteThreadLocMap.computeIfAbsent(Thread.currentThread().getId(),
+ id -> new DelayedDirtyPageWrite(flushDirtyPage, byteBufThreadLoc, pageSize, this));
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/d83f1ec0/modules/core/src/test/java/org/apache/ignite/internal/MemoryLeaksOnRestartNodeTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/MemoryLeaksOnRestartNodeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/MemoryLeaksOnRestartNodeTest.java
index 56ab091..df055fe 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/MemoryLeaksOnRestartNodeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/MemoryLeaksOnRestartNodeTest.java
@@ -18,7 +18,6 @@
package org.apache.ignite.internal;
import java.io.File;
-import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
@@ -76,8 +75,6 @@ public class MemoryLeaksOnRestartNodeTest extends GridCommonAbstractTest {
* @throws Exception On failed.
*/
public void test() throws Exception {
- System.setProperty(IgniteSystemProperties.IGNITE_DELAYED_REPLACED_PAGE_WRITE, "false");
-
// Warmup
for (int i = 0; i < RESTARTS / 2; ++i) {
startGrids(NODES);