You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ir...@apache.org on 2019/04/11 14:21:20 UTC
[ignite] branch master updated: IGNITE-11392 Improve LRT diagnostic
messages
This is an automated email from the ASF dual-hosted git repository.
irakov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 0cd72d2 IGNITE-11392 Improve LRT diagnostic messages
0cd72d2 is described below
commit 0cd72d22a2793805ecb5a554922bc74062261027
Author: denis-chudov <dc...@gridgain.com>
AuthorDate: Thu Apr 11 17:20:50 2019 +0300
IGNITE-11392 Improve LRT diagnostic messages
Signed-off-by: Ivan Rakov <ir...@apache.org>
---
.../org/apache/ignite/IgniteSystemProperties.java | 7 +
.../ignite/internal/TransactionsMXBeanImpl.java | 10 ++
.../cache/FetchActiveTxOwnerTraceClosure.java | 79 ++++++++
.../cache/GridCachePartitionExchangeManager.java | 142 ++++++++++++++-
.../processors/cache/GridCacheProcessor.java | 14 ++
.../TxOwnerDumpRequestAllowedSettingClosure.java | 56 ++++++
.../cache/transactions/IgniteTxManager.java | 31 ++++
.../apache/ignite/mxbean/TransactionsMXBean.java | 32 ++++
...CacheLongRunningTransactionDiagnosticsTest.java | 200 +++++++++++++++++++++
.../ignite/testsuites/IgniteCacheTestSuite9.java | 3 +
10 files changed, 566 insertions(+), 8 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index f6eb650..800ca56 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -1171,6 +1171,13 @@ public final class IgniteSystemProperties {
public static final String IGNITE_SQL_SYSTEM_SCHEMA_NAME_IGNITE = "IGNITE_SQL_SYSTEM_SCHEMA_NAME_IGNITE";
/**
+ * Shows if dump requests from local node to near node are allowed, when long running transaction
+ * is found. If allowed, the compute request to near node will be made to get thread dump of transaction
+ * owner thread.
+ */
+ public static final String IGNITE_TX_OWNER_DUMP_REQUESTS_ALLOWED = "IGNITE_TX_OWNER_DUMP_REQUESTS_ALLOWED";
+
+ /**
* Enforces singleton.
*/
private IgniteSystemProperties() {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java
index a488cb3..e70b9cc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java
@@ -132,6 +132,16 @@ public class TransactionsMXBeanImpl implements TransactionsMXBean {
}
/** {@inheritDoc} */
+ @Override public boolean getTxOwnerDumpRequestsAllowed() {
+ return ctx.cache().context().tm().txOwnerDumpRequestsAllowed();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setTxOwnerDumpRequestsAllowed(boolean allowed) {
+ ctx.cache().setTxOwnerDumpRequestsAllowed(allowed);
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
return S.toString(TransactionsMXBeanImpl.class, this);
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/FetchActiveTxOwnerTraceClosure.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/FetchActiveTxOwnerTraceClosure.java
new file mode 100644
index 0000000..0c51c83
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/FetchActiveTxOwnerTraceClosure.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.lang.IgniteCallable;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+
+/**
+ * Closure that is computed on near node to get the stack trace of active transaction owner thread.
+ */
+public class FetchActiveTxOwnerTraceClosure implements IgniteCallable<String> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private static final StackTraceElement[] STACK_TRACE_ELEMENT_EMPTY = new StackTraceElement[0];
+
+ /** */
+ private final long txOwnerThreadId;
+
+ /** */
+ public FetchActiveTxOwnerTraceClosure(long txOwnerThreadId) {
+ this.txOwnerThreadId = txOwnerThreadId;
+ }
+
+ /**
+ * Builds the stack trace dump of the transaction owner thread
+ *
+ * @return stack trace dump string
+ * @throws Exception If failed
+ */
+ @Override public String call() throws Exception {
+ StringBuilder traceDump = new StringBuilder("Stack trace of the transaction owner thread:\n");
+
+ for (StackTraceElement stackTraceElement : getStackTrace()) {
+ traceDump.append(stackTraceElement.toString());
+ traceDump.append("\n");
+ }
+
+ return traceDump.toString();
+ }
+
+ /**
+ * Gets the stack trace of the transaction owner thread
+ *
+ * @return stack trace elements
+ */
+ private StackTraceElement[] getStackTrace() {
+ ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
+
+ ThreadInfo threadInfo;
+
+ try {
+ threadInfo = threadMXBean.getThreadInfo(txOwnerThreadId, Integer.MAX_VALUE);
+ }
+ catch (SecurityException | IllegalArgumentException ignored) {
+ threadInfo = null;
+ }
+
+ return threadInfo == null ? STACK_TRACE_ELEMENT_EMPTY : threadInfo.getStackTrace();
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 8f6132e..324cc79 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -30,6 +30,7 @@ import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Objects;
+import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
@@ -39,11 +40,14 @@ import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteCompute;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.cache.affinity.AffinityFunction;
@@ -121,9 +125,12 @@ import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.internal.util.worker.GridWorker;
import org.apache.ignite.lang.IgniteBiInClosure;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgniteProductVersion;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.thread.IgniteThread;
+import org.apache.ignite.transactions.TransactionState;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -180,6 +187,9 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
private final AtomicLong lastRefresh = new AtomicLong(-1);
/** */
+ private final ActionLimiter<IgniteInternalTx> ltrDumpLimiter = new ActionLimiter<>(1);
+
+ /** */
@GridToStringInclude
private ExchangeWorker exchWorker;
@@ -1989,17 +1999,24 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
WarningsGroup warnings = new WarningsGroup("First %d long running transactions [total=%d]",
diagnosticLog, DIAGNOSTIC_WARN_LIMIT);
- for (IgniteInternalTx tx : tm.activeTransactions()) {
- if (curTime - tx.startTime() > timeout) {
- found = true;
+ synchronized (ltrDumpLimiter) {
+ for (IgniteInternalTx tx : tm.activeTransactions()) {
+ if (curTime - tx.startTime() > timeout) {
+ found = true;
+
+ if (warnings.canAddMessage()) {
+ warnings.add(">>> Transaction [startTime=" + formatTime(tx.startTime()) +
+ ", curTime=" + formatTime(curTime) + ", tx=" + tx + ']');
- if (warnings.canAddMessage()) {
- warnings.add(">>> Transaction [startTime=" + formatTime(tx.startTime()) +
- ", curTime=" + formatTime(curTime) + ", tx=" + tx + ']');
+ if (ltrDumpLimiter.allowAction(tx))
+ dumpLongRunningTransaction(tx);
+ }
+ else
+ warnings.incTotal();
}
- else
- warnings.incTotal();
}
+
+ ltrDumpLimiter.trim();
}
warnings.printToLog();
@@ -2066,6 +2083,62 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
}
/**
+ * Dumps long running transaction. If transaction is active and is not near, sends compute request
+ * to near node to get the stack trace of transaction owner thread.
+ *
+ * @param tx Transaction.
+ */
+ private void dumpLongRunningTransaction(IgniteInternalTx tx) {
+ if (cctx.tm().txOwnerDumpRequestsAllowed() && tx.local() && tx.state() == TransactionState.ACTIVE) {
+ Collection<UUID> masterNodeIds = tx.masterNodeIds();
+
+ if (masterNodeIds.size() == 1) {
+ UUID nearNodeId = masterNodeIds.iterator().next();
+
+ long txOwnerThreadId = tx.threadId();
+
+ Ignite ignite = cctx.kernalContext().grid();
+
+ IgniteCompute compute = ignite.compute(ignite.cluster().forNodeId(nearNodeId));
+
+ try {
+ compute
+ .callAsync(new FetchActiveTxOwnerTraceClosure(txOwnerThreadId))
+ .listen(new IgniteInClosure<IgniteFuture<String>>() {
+ @Override public void apply(IgniteFuture<String> strIgniteFut) {
+ String traceDump = null;
+
+ try {
+ traceDump = strIgniteFut.get();
+ }
+ catch (Exception e) {
+ U.warn(
+ diagnosticLog,
+ "Could not get thread dump from transaction owner near node: " + e.getMessage()
+ );
+ }
+
+ if (traceDump != null) {
+ U.warn(
+ diagnosticLog,
+ String.format(
+ "Dumping the near node thread that started transaction [xidVer=%s]\n%s",
+ tx.xidVersion().toString(),
+ traceDump
+ )
+ );
+ }
+ }
+ });
+ }
+ catch (Exception e) {
+ U.warn(diagnosticLog, "Could not send dump request to transaction owner near node: " + e.getMessage());
+ }
+ }
+ }
+ }
+
+ /**
* @param timeout Operation timeout.
*/
public void dumpLongRunningOperations(long timeout) {
@@ -3521,4 +3594,57 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
}
}
}
+
+ /**
+ * Class to limit action count for unique objects.
+ * <p>
+ * NO guarantees of thread safety are provided.
+ */
+ private static class ActionLimiter<T> {
+ /** */
+ private final int limit;
+
+ /**
+ * Internal storage of objects and counters for each of object.
+ */
+ private final Map<T, AtomicInteger> actionsCnt = new HashMap<>();
+
+ /**
+ * Set of active objects.
+ */
+ private final Set<T> activeObjects = new HashSet<>();
+
+ /**
+ * @param limit Limit.
+ */
+ private ActionLimiter(int limit) {
+ this.limit = limit;
+ }
+
+ /**
+ * Shows if action is allowed for the given object. Adds this object to internal set of active
+ * objects that are still in use.
+ *
+ * @param obj object.
+ */
+ boolean allowAction(T obj) {
+ activeObjects.add(obj);
+
+ int cnt = actionsCnt.computeIfAbsent(obj, o -> new AtomicInteger(0))
+ .incrementAndGet();
+
+ return (cnt <= limit);
+ }
+
+ /**
+ * Removes old objects from limiter's internal storage. All objects that are contained in internal
+ * storage but not in set of active objects, are assumed as 'old'. This method is to be called
+ * after processing of collection of objects to purge limiter's internal storage.
+ */
+ void trim() {
+ actionsCnt.keySet().removeIf(key -> !activeObjects.contains(key));
+
+ activeObjects.clear();
+ }
+ }
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index e24764d..fb15175 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -42,6 +42,7 @@ import javax.cache.expiry.EternalExpiryPolicy;
import javax.cache.expiry.ExpiryPolicy;
import javax.management.MBeanServer;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteCompute;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.cache.CacheExistsException;
@@ -5703,6 +5704,19 @@ public class GridCacheProcessor extends GridProcessorAdapter {
}
/**
+ * Sets if dump requests from local node to near node are allowed, when long running transaction
+ * is found. If allowed, the compute request to near node will be made to get thread dump of transaction
+ * owner thread. Also broadcasts this setting on other server nodes in cluster.
+ *
+ * @param allowed whether allowed
+ */
+ public void setTxOwnerDumpRequestsAllowed(boolean allowed) {
+ IgniteCompute compute = ctx.grid().compute(ctx.grid().cluster().forServers());
+
+ compute.broadcast(new TxOwnerDumpRequestAllowedSettingClosure(allowed));
+ }
+
+ /**
* Recovery lifecycle for caches.
*/
private class CacheRecoveryLifecycle implements MetastorageLifecycleListener, DatabaseLifecycleListener {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/TxOwnerDumpRequestAllowedSettingClosure.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/TxOwnerDumpRequestAllowedSettingClosure.java
new file mode 100644
index 0000000..882aa89
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/TxOwnerDumpRequestAllowedSettingClosure.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+
+/**
+ * Closure that is sent on all server nodes in order to change transaction configuration parameter
+ * that allows or disallows dump requests from local to near nodes while detection long running
+ * transactions.
+ */
+public class TxOwnerDumpRequestAllowedSettingClosure implements IgniteRunnable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private final boolean allowed;
+
+ /**
+ * Auto-inject Ignite instance
+ */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** */
+ public TxOwnerDumpRequestAllowedSettingClosure(boolean allowed) {
+ this.allowed = allowed;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ ((IgniteEx)ignite)
+ .context()
+ .cache()
+ .context()
+ .tm()
+ .setTxOwnerDumpRequestsAllowed(allowed);
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index 94ad815..15776e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -103,6 +103,7 @@ import static org.apache.ignite.IgniteSystemProperties.IGNITE_DEFERRED_ONE_PHASE
import static org.apache.ignite.IgniteSystemProperties.IGNITE_MAX_COMPLETED_TX_COUNT;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_SLOW_TX_WARN_TIMEOUT;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_TX_DEADLOCK_DETECTION_MAX_ITERS;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_TX_OWNER_DUMP_REQUESTS_ALLOWED;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_TX_SALVAGE_TIMEOUT;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS;
import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
@@ -184,6 +185,14 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
/** TX handler. */
private IgniteTxHandler txHnd;
+ /**
+ * Shows if dump requests from local node to near node are allowed, when long running transaction
+ * is found. If allowed, the compute request to near node will be made to get thread dump of transaction
+ * owner thread.
+ */
+ private boolean txOwnerDumpRequestsAllowed =
+ IgniteSystemProperties.getBoolean(IGNITE_TX_OWNER_DUMP_REQUESTS_ALLOWED, true);
+
/** Committed local transactions. */
private final GridBoundedConcurrentOrderedMap<GridCacheVersion, Boolean> completedVersSorted =
new GridBoundedConcurrentOrderedMap<>(
@@ -378,6 +387,28 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
}
/**
+ * Sets if dump requests from local node to near node are allowed, when long running transaction
+ * is found. If allowed, the compute request to near node will be made to get thread dump of transaction
+ * owner thread.
+ *
+ * @return <code>true</code> if allowed, <code>false</code> otherwise.
+ */
+ public boolean txOwnerDumpRequestsAllowed() {
+ return txOwnerDumpRequestsAllowed;
+ }
+
+ /**
+ * Sets if dump requests from local node to near node are allowed, when long running transaction
+ * is found. If allowed, the compute request to near node will be made to get thread dump of transaction
+ * owner thread.
+ *
+ * @param allowed whether allowed
+ */
+ public void setTxOwnerDumpRequestsAllowed(boolean allowed) {
+ txOwnerDumpRequestsAllowed = allowed;
+ }
+
+ /**
* Invalidates transaction.
*
* @param tx Transaction.
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/TransactionsMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/TransactionsMXBean.java
index 2a125fa..eef7931 100644
--- a/modules/core/src/main/java/org/apache/ignite/mxbean/TransactionsMXBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/TransactionsMXBean.java
@@ -96,4 +96,36 @@ public interface TransactionsMXBean {
"Transaction timeout on partition map exchange in milliseconds."
)
public void setTxTimeoutOnPartitionMapExchange(long timeout);
+
+ /**
+ * Shows if dump requests from local node to near node are allowed, when long running transaction
+ * is found. If allowed, the compute request to near node will be made to get thread dump of transaction
+ * owner thread.
+ *
+ * @return <code>true</code> if allowed, <code>false</code> otherwise.
+ */
+ @MXBeanDescription(
+ "Shows if dump requests from local node to near node are allowed, " +
+ "when long running transaction is found. If allowed, the compute request to near " +
+ "node will be made to get thread dump of transaction owner thread."
+ )
+ public boolean getTxOwnerDumpRequestsAllowed();
+
+ /**
+ * Sets if dump requests from local node to near node are allowed, when long running transaction
+ * is found. If allowed, the compute request to near node will be made to get thread dump of transaction
+ * owner thread.
+ *
+ * @param allowed whether to allow
+ */
+ @MXBeanDescription(
+ "Shows if dump requests from local node to near node are allowed, " +
+ "when long running transaction is found. If allowed, the compute request to near " +
+ "node will be made to get thread dump of transaction owner thread."
+ )
+ @MXBeanParametersNames("allowed")
+ @MXBeanParametersDescriptions(
+ "whether to allow"
+ )
+ public void setTxOwnerDumpRequestsAllowed(boolean allowed);
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheLongRunningTransactionDiagnosticsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheLongRunningTransactionDiagnosticsTest.java
new file mode 100644
index 0000000..00ac8b5
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheLongRunningTransactionDiagnosticsTest.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache;
+
+import java.lang.management.ManagementFactory;
+import javax.management.MBeanServer;
+import javax.management.MBeanServerInvocationHandler;
+import javax.management.ObjectName;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.GridJobExecuteRequest;
+import org.apache.ignite.internal.GridTopic;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.TransactionsMXBeanImpl;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.mxbean.TransactionsMXBean;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.junit.Test;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ * Test class for diagnostics of long running transactions.
+ */
+public class GridCacheLongRunningTransactionDiagnosticsTest extends GridCommonAbstractTest {
+ /** */
+ private static final long LONG_OP_TIMEOUT = 500;
+
+ /** */
+ private static final long TX_TIMEOUT = LONG_OP_TIMEOUT * 2;
+
+ /** */
+ private static final String CACHE_NAME = "test";
+
+ /** */
+ private static String longOpTimeoutCommon;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setConsistentId(igniteInstanceName);
+
+ cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+ boolean isClient = "client".equals(igniteInstanceName);
+
+ cfg.setClientMode(isClient);
+
+ if (!isClient) {
+ CacheConfiguration ccfg = new CacheConfiguration(CACHE_NAME);
+
+ ccfg.setAtomicityMode(TRANSACTIONAL);
+ ccfg.setBackups(2);
+ ccfg.setWriteSynchronizationMode(FULL_SYNC);
+
+ cfg.setCacheConfiguration(ccfg);
+ }
+
+ return cfg;
+ }
+
+ /**
+ * Setting long op timeout to small value to make this tests faster
+ */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ longOpTimeoutCommon = System.getProperty(IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT);
+
+ System.setProperty(IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT, String.valueOf(LONG_OP_TIMEOUT));
+ }
+
+ /**
+ * Returning long operations timeout to its former value.
+ */
+ @Override protected void afterTestsStopped() throws Exception {
+ if (longOpTimeoutCommon != null)
+ System.setProperty(IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT, longOpTimeoutCommon);
+ else
+ System.clearProperty(IGNITE_LONG_OPERATIONS_DUMP_TIMEOUT);
+
+ super.afterTestsStopped();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ super.afterTest();
+ }
+
+ /**
+ * Tests long transaction scenario.
+ *
+ * @throws Exception if grids start failed.
+ */
+ @Test
+ public void testLrt() throws Exception {
+ startGridsMultiThreaded(2);
+
+ imitateLongTransaction(true);
+ }
+
+ /**
+ * Tests transaction mx bean and its ability to turn on and off thread dump requests
+ * from local node to near node.
+ *
+ * @throws Exception if grids start failed.
+ */
+ @Test
+ public void testLrtChangeSetting() throws Exception {
+ startGridsMultiThreaded(2);
+
+ TransactionsMXBean tMXBean0 = txMXBean(0);
+ TransactionsMXBean tMXBean1 = txMXBean(1);
+
+ assertTrue(tMXBean0.getTxOwnerDumpRequestsAllowed());
+ assertTrue(tMXBean1.getTxOwnerDumpRequestsAllowed());
+
+ tMXBean0.setTxOwnerDumpRequestsAllowed(false);
+
+ assertFalse(tMXBean0.getTxOwnerDumpRequestsAllowed());
+ assertFalse(tMXBean1.getTxOwnerDumpRequestsAllowed());
+
+ imitateLongTransaction(false);
+ }
+
+ /**
+ * Creates a client node and imitates a long running transaction by this client.
+ *
+ * @param shouldRcvThreadDumpReq whether client node (transaction owner) should
+ * receive dump request from local node.
+ * @throws Exception if failed.
+ */
+ private void imitateLongTransaction(boolean shouldRcvThreadDumpReq) throws Exception {
+ final int val = 0;
+
+ final IgniteEx client = startGrid("client");
+
+ assertTrue(client.configuration().isClientMode());
+
+ client.getOrCreateCache(CACHE_NAME);
+
+ StringBuilder taskNameContainer = new StringBuilder();
+
+ client.context().io().addMessageListener(
+ GridTopic.TOPIC_JOB,
+ (nodeId, msg, plc) -> taskNameContainer.append(((GridJobExecuteRequest) msg).getTaskName())
+ );
+
+ try (Transaction tx = client.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, TX_TIMEOUT, 1)) {
+ client.cache(CACHE_NAME).put(val, 0);
+
+ doSleep(TX_TIMEOUT * 2);
+
+ tx.rollback();
+ }
+
+ assertEquals(
+ shouldRcvThreadDumpReq,
+ FetchActiveTxOwnerTraceClosure.class.getName().equals(taskNameContainer.toString())
+ );
+ }
+
+ /**
+ *
+ */
+ private TransactionsMXBean txMXBean(int igniteInt) throws Exception {
+ ObjectName mbeanName = U.makeMBeanName(getTestIgniteInstanceName(igniteInt), "Transactions",
+ TransactionsMXBeanImpl.class.getSimpleName());
+
+ MBeanServer mbeanSrv = ManagementFactory.getPlatformMBeanServer();
+
+ if (!mbeanSrv.isRegistered(mbeanName))
+ fail("MBean is not registered: " + mbeanName.getCanonicalName());
+
+ return MBeanServerInvocationHandler.newProxyInstance(mbeanSrv, mbeanName, TransactionsMXBean.class, true);
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java
index 5e31aa8..3618ca6 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.ignite.internal.processors.cache.CachePutIfAbsentTest;
+import org.apache.ignite.internal.processors.cache.GridCacheLongRunningTransactionDiagnosticsTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheGetCustomCollectionsSelfTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheLoadRebalanceEvictionSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheAtomicPrimarySyncBackPressureTest;
@@ -78,6 +79,8 @@ public class IgniteCacheTestSuite9 {
GridTestUtils.addTestIfNeeded(suite, IoStatisticsManagerSelfTest.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite, IoStatisticsMetricsLocalMXBeanImplSelfTest.class, ignoredTests);
+ GridTestUtils.addTestIfNeeded(suite, GridCacheLongRunningTransactionDiagnosticsTest.class, ignoredTests);
+
return suite;
}
}