You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by as...@apache.org on 2020/08/21 15:08:19 UTC
[ignite] branch master updated: IGNITE-13363 Prevent
GridDhtCacheEntry.toString wait for entry lock forever to avoid deadlocks.
This is an automated email from the ASF dual-hosted git repository.
ascherbakov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new f4b30f7 IGNITE-13363 Prevent GridDhtCacheEntry.toString wait for entry lock forever to avoid deadlocks.
f4b30f7 is described below
commit f4b30f7f1e736845ffa8eaf2d8aa171700a928eb
Author: Alexey Scherbakov <al...@gmail.com>
AuthorDate: Fri Aug 21 18:02:41 2020 +0300
IGNITE-13363 Prevent GridDhtCacheEntry.toString wait for entry lock forever to avoid deadlocks.
---
.../org/apache/ignite/internal/IgnitionEx.java | 22 ++
.../processors/cache/GridCacheEntryEx.java | 11 +
.../processors/cache/GridCacheMapEntry.java | 57 ++++-
.../distributed/GridDistributedCacheEntry.java | 9 +-
.../cache/distributed/dht/GridDhtCacheEntry.java | 13 +-
.../dht/colocated/GridDhtDetachedCacheEntry.java | 9 +-
.../cache/distributed/near/GridNearCacheEntry.java | 9 +-
.../cache/local/GridLocalCacheEntry.java | 9 +-
.../processors/resource/DependencyResolver.java | 32 +++
.../processors/resource/GridResourceProcessor.java | 19 ++
.../resource/NoopDependencyResolver.java | 28 +++
.../spi/communication/tcp/TcpCommunicationSpi.java | 26 ++-
.../tcp/internal/ConnectionClientPool.java | 3 +-
.../processors/cache/GridCacheTestEntryEx.java | 5 +
.../TxDeadlockOnEntryToStringTest.java | 246 +++++++++++++++++++++
...unicationInverseConnectionEstablishingTest.java | 4 +-
.../AbstractTestDependencyResolver.java | 55 +++++
.../testframework/TestDependencyResolver.java | 48 ++++
.../testframework/junits/GridAbstractTest.java | 39 ++++
.../testsuites/TxDeadlockDetectionTestSuite.java | 4 +-
20 files changed, 585 insertions(+), 63 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index 436eb15..66a2cda 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -87,6 +87,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.topology.Grid
import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
import org.apache.ignite.internal.processors.metastorage.DistributedMetaStorage;
import org.apache.ignite.internal.processors.metastorage.persistence.DistributedMetaStorageImpl;
+import org.apache.ignite.internal.processors.resource.DependencyResolver;
import org.apache.ignite.internal.processors.resource.GridSpringResourceContext;
import org.apache.ignite.internal.util.GridConcurrentHashSet;
import org.apache.ignite.internal.util.IgniteUtils;
@@ -226,6 +227,9 @@ public class IgnitionEx {
/** */
private static ThreadLocal<Boolean> clientMode = new ThreadLocal<>();
+ /** Dependency container. */
+ private static ThreadLocal<DependencyResolver> dependencyResolver = new ThreadLocal<>();
+
/**
* Enforces singleton.
*/
@@ -1459,6 +1463,24 @@ public class IgnitionEx {
}
/**
+ * Sets custom dependency resolver which provides overridden dependencies
+ *
+ * @param rslvr Dependency resolver.
+ */
+ public static void dependencyResolver(DependencyResolver rslvr) {
+ dependencyResolver.set(rslvr);
+ }
+
+ /**
+ * Custom dependency resolver.
+ *
+ * @return Returns {@code null} if resolver wasn't added.
+ */
+ public static DependencyResolver dependencyResolver() {
+ return dependencyResolver.get();
+ }
+
+ /**
* Start context encapsulates all starting parameters.
*/
private static final class GridStartContext {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
index 63a6ee8..fb68256 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
@@ -1204,6 +1204,17 @@ public interface GridCacheEntryEx {
public void unlockEntry();
/**
+ * Locks entry to protect from concurrent access. Intended to be used instead of inherent java synchronization. This
+ * allows to separate locking from unlocking in time and/or code units.
+ *
+ * @param timeout period of waiting in millis;
+ * @return {@code true} if the lock was free and was acquired by the current thread, or the lock was already held by
+ * the current thread; and {@code false} if the waiting time elapsed before the lock could be acquired
+ * @see GridCacheEntryEx#unlockEntry().
+ */
+ public boolean tryLockEntry(long timeout);
+
+ /**
* Tests whether the entry is locked currently.
*
* @see GridCacheEntryEx#lockEntry().
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index e92c00e..d157283 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -24,9 +24,11 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Supplier;
import javax.cache.Cache;
import javax.cache.expiry.ExpiryPolicy;
import javax.cache.processor.EntryProcessor;
@@ -91,6 +93,7 @@ import org.apache.ignite.internal.util.lang.GridCursor;
import org.apache.ignite.internal.util.lang.GridMetadataAwareAdapter;
import org.apache.ignite.internal.util.lang.GridTuple;
import org.apache.ignite.internal.util.lang.GridTuple3;
+import org.apache.ignite.internal.util.tostring.GridToStringBuilder;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.F;
@@ -107,6 +110,7 @@ import org.apache.ignite.thread.IgniteThread;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
+import static org.apache.ignite.IgniteSystemProperties.getLong;
import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_EXPIRED;
import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_LOCKED;
import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_PUT;
@@ -132,6 +136,15 @@ import static org.apache.ignite.internal.processors.dr.GridDrType.DR_NONE;
@SuppressWarnings({"TooBroadScope"})
public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter implements GridCacheEntryEx {
/** */
+ public static final GridCacheAtomicVersionComparator ATOMIC_VER_COMPARATOR = new GridCacheAtomicVersionComparator();
+
+ /** Property name for entry lock timeout. */
+ public static final String ENTRY_LOCK_TIMEOUT_ENV = "ENTRY_LOCK_TIMEOUT";
+
+ /** Entry lock time awaiting. */
+ private static final long ENTRY_LOCK_TIMEOUT = getLong(ENTRY_LOCK_TIMEOUT_ENV, 1000);
+
+ /** */
private static final byte IS_DELETED_MASK = 0x01;
/** */
@@ -140,9 +153,6 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
/** */
private static final byte IS_EVICT_DISABLED = 0x04;
- /** */
- public static final GridCacheAtomicVersionComparator ATOMIC_VER_COMPARATOR = new GridCacheAtomicVersionComparator();
-
/**
* NOTE
* <br/>
@@ -5039,6 +5049,18 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
}
/** {@inheritDoc} */
+ @Override public boolean tryLockEntry(long timeout) {
+ try {
+ return lock.tryLock(timeout, TimeUnit.MILLISECONDS);
+ }
+ catch (InterruptedException ignite) {
+ Thread.currentThread().interrupt();
+
+ return false;
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public void unlockEntry() {
lock.unlock();
}
@@ -5086,13 +5108,32 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
/** {@inheritDoc} */
@Override public String toString() {
- lockEntry();
+ return toStringWithTryLock(() -> S.toString(GridCacheMapEntry.class, this));
+ }
- try {
- return S.toString(GridCacheMapEntry.class, this);
+ /**
+ * Does thread safe {@link #toString} for {@link GridCacheMapEntry} classes.
+ *
+ * @param dfltToStr {@link #toString()} supplier.
+ * @return Result of dfltToStr call If lock acquired or a short representation of {@link GridCacheMapEntry}.
+ */
+ protected String toStringWithTryLock(Supplier<String> dfltToStr) {
+ if (tryLockEntry(ENTRY_LOCK_TIMEOUT)) {
+ try {
+ return dfltToStr.get();
+ }
+ finally {
+ unlockEntry();
+ }
}
- finally {
- unlockEntry();
+ else {
+ String keySens = GridToStringBuilder.includeSensitive() ? ", key=" + key : "";
+
+ return "GridCacheMapEntry [err='Partial result represented because entry lock wasn't acquired."
+ + " Waiting time elapsed.'"
+ + keySens
+ + ", hash=" + hash
+ + "]";
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheEntry.java
index 7a8bab4..aa7bfcb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheEntry.java
@@ -787,13 +787,6 @@ public class GridDistributedCacheEntry extends GridCacheMapEntry {
/** {@inheritDoc} */
@Override public String toString() {
- lockEntry();
-
- try {
- return S.toString(GridDistributedCacheEntry.class, this, super.toString());
- }
- finally {
- unlockEntry();
- }
+ return toStringWithTryLock(() -> S.toString(GridDistributedCacheEntry.class, this, super.toString()));
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
index a9ce5a1..f66602d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
@@ -841,16 +841,9 @@ public class GridDhtCacheEntry extends GridDistributedCacheEntry {
/** {@inheritDoc} */
@Override public String toString() {
- lockEntry();
-
- try {
- return S.toString(GridDhtCacheEntry.class, this,
- "part", locPart.id(),
- "super", super.toString());
- }
- finally {
- unlockEntry();
- }
+ return toStringWithTryLock(() -> S.toString(GridDhtCacheEntry.class, this,
+ "part", locPart.id(),
+ "super", super.toString()));
}
/** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
index 346a992..68caf44 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
@@ -95,14 +95,7 @@ public class GridDhtDetachedCacheEntry extends GridDistributedCacheEntry {
/** {@inheritDoc} */
@Override public String toString() {
- lockEntry();
-
- try {
- return S.toString(GridDhtDetachedCacheEntry.class, this, "super", super.toString());
- }
- finally {
- unlockEntry();
- }
+ return toStringWithTryLock(() -> S.toString(GridDhtDetachedCacheEntry.class, this, "super", super.toString()));
}
/** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
index 0db9249..d4f0b1a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
@@ -791,13 +791,6 @@ public class GridNearCacheEntry extends GridDistributedCacheEntry {
/** {@inheritDoc} */
@Override public String toString() {
- lockEntry();
-
- try {
- return S.toString(GridNearCacheEntry.class, this, "super", super.toString());
- }
- finally {
- unlockEntry();
- }
+ return toStringWithTryLock(() -> S.toString(GridNearCacheEntry.class, this, "super", super.toString()));
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCacheEntry.java
index dfe9d85..f716759 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCacheEntry.java
@@ -377,13 +377,6 @@ public class GridLocalCacheEntry extends GridCacheMapEntry {
/** {@inheritDoc} */
@Override public String toString() {
- lockEntry();
-
- try {
- return S.toString(GridLocalCacheEntry.class, this, super.toString());
- }
- finally {
- unlockEntry();
- }
+ return toStringWithTryLock(() -> S.toString(GridLocalCacheEntry.class, this, super.toString()));
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/DependencyResolver.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/DependencyResolver.java
new file mode 100644
index 0000000..cd905e3
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/DependencyResolver.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.resource;
+
+/**
+ * The interface specifies a container of dependencies.
+ */
+public interface DependencyResolver {
+ /**
+ * The method doing resolve input dependency and return original or override class.
+ *
+ * @param instance Input dependency.
+ *
+ * @return Original instance or override.
+ */
+ <T> T resolve(T instance);
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java
index 461b8d9..891eeb1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/GridResourceProcessor.java
@@ -32,6 +32,7 @@ import org.apache.ignite.internal.GridInternalWrapper;
import org.apache.ignite.internal.GridJobContextImpl;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.GridTaskSessionImpl;
+import org.apache.ignite.internal.IgnitionEx;
import org.apache.ignite.internal.managers.deployment.GridDeployment;
import org.apache.ignite.internal.processors.GridProcessorAdapter;
import org.apache.ignite.internal.util.typedef.X;
@@ -57,6 +58,9 @@ public class GridResourceProcessor extends GridProcessorAdapter {
/** */
private final GridResourceInjector[] injectorByAnnotation;
+ /** Dependency container. */
+ private volatile DependencyResolver dependencyResolver = new NoopDependencyResolver();
+
/**
* Creates resources processor.
*
@@ -77,6 +81,11 @@ public class GridResourceProcessor extends GridProcessorAdapter {
/** {@inheritDoc} */
@Override public void start() throws IgniteCheckedException {
+ final DependencyResolver extRslvr = IgnitionEx.dependencyResolver();
+
+ if (extRslvr != null)
+ this.dependencyResolver = extRslvr;
+
if (log.isDebugEnabled())
log.debug("Started resource processor.");
}
@@ -568,4 +577,14 @@ public class GridResourceProcessor extends GridProcessorAdapter {
ioc.printMemoryStats();
}
+
+ /**
+ * Delegates resource resolving to the provided dependency resolver, which wraps passed instance if necessary.
+ *
+ * @param instance Instance of delegated class.
+ * @return Original instance or wrapped if wrapper exists.
+ */
+ public <T> T resolve(T instance) {
+ return dependencyResolver.resolve(instance);
+ }
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/NoopDependencyResolver.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/NoopDependencyResolver.java
new file mode 100644
index 0000000..9c3af65
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/resource/NoopDependencyResolver.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.resource;
+
+/**
+ * Noop implementation of dependency resolver. It's used by default.
+ */
+public class NoopDependencyResolver implements DependencyResolver {
+ /** {@inheritDoc} */
+ @Override public <T> T resolve(T instance) {
+ return instance;
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 5732994..1012612 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -46,6 +46,7 @@ import org.apache.ignite.internal.managers.communication.GridIoMessage;
import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
import org.apache.ignite.internal.processors.failure.FailureProcessor;
import org.apache.ignite.internal.processors.metric.impl.MetricUtils;
+import org.apache.ignite.internal.processors.resource.GridResourceProcessor;
import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
import org.apache.ignite.internal.processors.tracing.MTC;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
@@ -677,7 +678,7 @@ public class TcpCommunicationSpi extends TcpCommunicationConfigInitializer {
else
connPlc = new FirstConnectionPolicy();
- this.srvLsnr = new InboundConnectionHandler(
+ this.srvLsnr = resolve(ignite, new InboundConnectionHandler(
log,
cfg,
nodeGetter,
@@ -703,12 +704,11 @@ public class TcpCommunicationSpi extends TcpCommunicationConfigInitializer {
lsnr.onDisconnected(nodeId);
}
}
- );
+ ));
- GridTimeoutProcessor timeoutProcessor = ignite instanceof IgniteKernal ?
- ((IgniteKernal)ignite).context().timeout() : null;
+ GridTimeoutProcessor timeoutProcessor = ignite instanceof IgniteKernal ? ((IgniteKernal)ignite).context().timeout() : null;
- this.nioSrvWrapper = new GridNioServerWrapper(
+ this.nioSrvWrapper = resolve(ignite, new GridNioServerWrapper(
log,
cfg,
timeoutProcessor,
@@ -741,11 +741,11 @@ public class TcpCommunicationSpi extends TcpCommunicationConfigInitializer {
((CommunicationListenerEx<Message>)lsnr).onChannelOpened(rmtNodeId, initMsg, channel);
}
}
- );
+ ));
this.srvLsnr.setNioSrvWrapper(nioSrvWrapper);
- this.clientPool = new ConnectionClientPool(
+ this.clientPool = resolve(ignite, new ConnectionClientPool(
cfg,
attributeNames,
log,
@@ -758,9 +758,10 @@ public class TcpCommunicationSpi extends TcpCommunicationConfigInitializer {
timeoutProcessor,
stateProvider,
nioSrvWrapper
- );
+ ));
+
+ this.srvLsnr.setClientPool(clientPool);
- srvLsnr.setClientPool(clientPool);
nioSrvWrapper.clientPool(clientPool);
discoLsnr = new CommunicationDiscoveryEventListener(clientPool, metricsLsnr);
@@ -1071,6 +1072,13 @@ public class TcpCommunicationSpi extends TcpCommunicationConfigInitializer {
}
/**
+ * Checks {@link Ignite} implementation type and calls {@link GridResourceProcessor#resolve(Object)} or returns original.
+ */
+ private <T> T resolve(Ignite ignite, T instance) {
+ return ignite instanceof IgniteKernal ? ((IgniteKernal)ignite).context().resource().resolve(instance) : instance;
+ }
+
+ /**
* Checks that node has specified attribute and prints warning if it does not.
*
* @param node Node to check.
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/ConnectionClientPool.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/ConnectionClientPool.java
index 897a9ed..14a0219 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/ConnectionClientPool.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/ConnectionClientPool.java
@@ -21,6 +21,7 @@ import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.Map;
+import java.util.Optional;
import java.util.StringJoiner;
import java.util.UUID;
import java.util.concurrent.ConcurrentMap;
@@ -681,7 +682,7 @@ public class ConnectionClientPool {
GridFutureAdapter<GridCommunicationClient> fut = clientFuts.remove(connKey);
if (nonNull(fut))
- fut.get().forceClose();
+ Optional.ofNullable(fut.get()).ifPresent(GridCommunicationClient::forceClose);
}
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
index 8518c7b..c406054 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
@@ -957,6 +957,11 @@ public class GridCacheTestEntryEx extends GridMetadataAwareAdapter implements Gr
}
/** {@inheritDoc} */
+ @Override public boolean tryLockEntry(long timeout) {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Override public boolean lockedByCurrentThread() {
return false;
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockOnEntryToStringTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockOnEntryToStringTest.java
new file mode 100644
index 0000000..a402ec3
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockOnEntryToStringTest.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.FailureType;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
+import org.apache.ignite.internal.processors.resource.DependencyResolver;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
+import org.apache.ignite.internal.util.UUIDCollectionMessage;
+import org.apache.ignite.internal.util.nio.GridNioSession;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationMetricsListener;
+import org.apache.ignite.spi.communication.tcp.internal.CommunicationWorker;
+import org.apache.ignite.spi.communication.tcp.internal.ConnectionClientPool;
+import org.apache.ignite.spi.communication.tcp.internal.GridNioServerWrapper;
+import org.apache.ignite.spi.communication.tcp.internal.InboundConnectionHandler;
+import org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage2;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.TestDependencyResolver;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+import org.junit.Test;
+
+/**
+ * GridDhtCacheEntry::toString leads to system "deadlock" on the timeoutWorker.
+ */
+public class TxDeadlockOnEntryToStringTest extends GridCommonAbstractTest {
+ /** Test key. */
+ private static final int TEST_KEY = 1;
+
+ /**
+ * Mark that incoming connect must be rejected.
+ */
+ private static final AtomicBoolean rejectHandshake = new AtomicBoolean(false);
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME).setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL));
+
+ return cfg;
+ }
+
+ /**
+ * We removed locks from toString on Entry. The case, a thread X lock entry, after that trying to do a handshake
+ * with connecting node. But, handshake fails on the first attempt. Between these events, timeout worked trying to
+ * print this entry, but I locked. The thread X can't reconnect, because timeout worker hangs.
+ */
+ @Test
+ public void testDeadlockOnTimeoutWorkerAndToString() throws Exception {
+ // Setup
+ TestDependencyResolver nearDepRslvr = new TestDependencyResolver();
+ IgniteEx nearNode = startGrid(0, nearDepRslvr);
+
+ TestDependencyResolver incomingDepRslvr = new TestDependencyResolver(this::resolve);
+ IgniteEx incomingNode = startGrid(1, incomingDepRslvr);
+
+ GridTimeoutProcessor tp = nearNode.context().timeout();
+ ConnectionClientPool pool = nearDepRslvr.getDependency(ConnectionClientPool.class);
+
+ GridCacheEntryEx ex = getEntry(nearNode, DEFAULT_CACHE_NAME, TEST_KEY);
+
+ // Act
+ try {
+ ex.lockEntry(); // Lock entry in current thread
+
+ // Print the entry from another thread via timeObject.
+ CountDownLatch entryPrinted = new CountDownLatch(1);
+ CountDownLatch entryReadyToPrint = new CountDownLatch(1);
+ tp.addTimeoutObject(new EntryPrinterTimeoutObject(ex, entryPrinted, entryReadyToPrint));
+
+ entryReadyToPrint.await();
+
+ // Try to do first handshake with hangs, after reconnect handshake should be passed.
+ rejectHandshake.set(true);
+
+ pool.forceCloseConnection(incomingNode.localNode().id());
+
+ nearNode.configuration().getCommunicationSpi().sendMessage(incomingNode.localNode(), UUIDCollectionMessage.of(UUID.randomUUID()));
+
+ // Check
+ assertTrue(GridTestUtils.waitForCondition(() -> entryPrinted.getCount() == 0, 5_000));
+ }
+ finally {
+ ex.unlockEntry();
+ }
+ }
+
+ /**
+ * @param node Node.
+ * @param cacheName Cache name.
+ * @param key Key.
+ */
+ private GridCacheEntryEx getEntry(IgniteEx node, String cacheName, Object key) {
+ return node.cachex(cacheName).context().cache()
+ .entryEx(
+ node.cachex(cacheName).context().toCacheKeyObject(key),
+ node.context().discovery().topologyVersionEx()
+ );
+ }
+
+ /**
+ * Call toString via time interval.
+ */
+ private class EntryPrinterTimeoutObject implements GridTimeoutObject {
+ /** Id. */
+ private final IgniteUuid id = IgniteUuid.randomUuid();
+
+ /** Entry. */
+ private final GridCacheEntryEx entry;
+
+ /** Entry printed. */
+ private final CountDownLatch entryPrinted;
+
+ /** Entry ready to print. */
+ private final CountDownLatch entryReadyToPrint;
+
+ /**
+ * @param entry Entry.
+ * @param entryPrinted Entry printed.
+ * @param entryReadyToPrint Entry ready to print.
+ */
+ private EntryPrinterTimeoutObject(GridCacheEntryEx entry, CountDownLatch entryPrinted,
+ CountDownLatch entryReadyToPrint) {
+ this.entry = entry;
+ this.entryPrinted = entryPrinted;
+ this.entryReadyToPrint = entryReadyToPrint;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteUuid timeoutId() {
+ return id;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long endTime() {
+ return 1;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onTimeout() {
+ entryReadyToPrint.countDown();
+
+ log.info(entry.toString());
+
+ entryPrinted.countDown();
+ }
+ }
+
+ /**
+ * The method reference implementation of {@link DependencyResolver}. It adds an additional behavior to {@link
+ * InboundConnectionHandler}.
+ *
+ * @param instance Delegated instance.
+ */
+ private <T> T resolve(T instance) {
+ if (instance instanceof InboundConnectionHandler) {
+ InboundConnectionHandler hnd = (InboundConnectionHandler)instance;
+
+ return (T)(new InboundConnectionHandler(null, null, null, null, null, null, null, null, null, null, null, null, null, false, null, null) {
+ @Override public void setNioSrvWrapper(GridNioServerWrapper nioSrvWrapper) {
+ hnd.setNioSrvWrapper(nioSrvWrapper);
+ }
+
+ @Override public void setClientPool(ConnectionClientPool pool) {
+ hnd.setClientPool(pool);
+ }
+
+ @Override public void onSessionWriteTimeout(GridNioSession ses) {
+ hnd.onSessionWriteTimeout(ses);
+ }
+
+ @Override public void onConnected(GridNioSession ses) {
+ hnd.onConnected(ses);
+ }
+
+ @Override public void onMessageSent(GridNioSession ses, Message msg) {
+ hnd.onMessageSent(ses, msg);
+ }
+
+ @Override public void onMessage(GridNioSession ses, Message msg) {
+ if (rejectHandshake.get() && msg instanceof HandshakeMessage2) {
+ rejectHandshake.set(false);
+
+ ses.close();
+
+ return;
+ }
+
+ hnd.onMessage(ses, msg);
+ }
+
+ @Override public void onFailure(FailureType failureType, Throwable failure) {
+ hnd.onFailure(failureType, failure);
+ }
+
+ @Override public void onDisconnected(GridNioSession ses, @Nullable Exception e) {
+ hnd.onDisconnected(ses, e);
+ }
+
+ @Override public void stop() {
+ hnd.stop();
+ }
+
+ @Override public void communicationWorker(CommunicationWorker commWorker) {
+ hnd.communicationWorker(commWorker);
+ }
+
+ @Override public void onSessionIdleTimeout(GridNioSession ses) {
+ hnd.onSessionIdleTimeout(ses);
+ }
+
+ @Override public void metricsListener(@Nullable TcpCommunicationMetricsListener metricsLsnr) {
+ hnd.metricsListener(metricsLsnr);
+ }
+ });
+ }
+
+ return instance;
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationInverseConnectionEstablishingTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationInverseConnectionEstablishingTest.java
index e2e4ebf..cbf4777 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationInverseConnectionEstablishingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationInverseConnectionEstablishingTest.java
@@ -250,7 +250,7 @@ public class GridTcpCommunicationInverseConnectionEstablishingTest extends GridC
for (int i = 0; i < SRVS_NUM; i++) {
ccfg = cacheConfiguration(CACHE_NAME, ATOMIC);
- startGrid(i, cfg -> {
+ startGrid(i, (UnaryOperator<IgniteConfiguration>) cfg -> {
ListeningTestLogger log = new ListeningTestLogger(false, cfg.getGridLogger());
log.registerListener(lsnr);
@@ -284,7 +284,7 @@ public class GridTcpCommunicationInverseConnectionEstablishingTest extends GridC
"Failed to wait for establishing inverse communication connection"
).build();
- startGrid(SRVS_NUM - 1, cfg -> {
+ startGrid(SRVS_NUM - 1, (UnaryOperator<IgniteConfiguration>) cfg -> {
ListeningTestLogger log = new ListeningTestLogger(false, cfg.getGridLogger());
log.registerListener(lsnr);
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/AbstractTestDependencyResolver.java b/modules/core/src/test/java/org/apache/ignite/testframework/AbstractTestDependencyResolver.java
new file mode 100644
index 0000000..dbaafd5
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/AbstractTestDependencyResolver.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testframework;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import org.apache.ignite.internal.processors.resource.DependencyResolver;
+
+/**
+ * The abstract implementation with registry inside, which provides a dependency on demand by {@link #getDependency(Class)}.
+ */
+public abstract class AbstractTestDependencyResolver implements DependencyResolver {
+ /** Dependency registry. */
+ private final ConcurrentMap<Class, Object> registry = new ConcurrentHashMap<>();
+
+ /** {@inheritDoc} */
+ @Override public <T> T resolve(T instance) {
+ T resolved = doResolve(instance);
+
+ registry.put(instance.getClass(), resolved);
+
+ return resolved;
+ }
+
+ /**
+ * Returns demanded dependency if it was registered.
+ *
+ * @param clazz Required type of dependency.
+ *
+ * @return Returns {@code null} if dependency wasn't registered.
+ */
+ public <T> T getDependency(Class<T> clazz) {
+ return (T) registry.get(clazz);
+ }
+
+ /**
+ * Custom resolver logic.
+ */
+ protected abstract <T> T doResolve(T instance);
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/TestDependencyResolver.java b/modules/core/src/test/java/org/apache/ignite/testframework/TestDependencyResolver.java
new file mode 100644
index 0000000..795fde5
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/TestDependencyResolver.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testframework;
+
+import org.apache.ignite.internal.processors.resource.DependencyResolver;
+import org.apache.ignite.internal.processors.resource.NoopDependencyResolver;
+
+/**
+ * Test implementation of dependency resolver. You can pass your logic via method reference.
+ */
+public class TestDependencyResolver extends AbstractTestDependencyResolver {
+ /** Resolver. */
+ private final DependencyResolver resolver;
+
+ /**
+ * @param resolver Method reference.
+ */
+ public TestDependencyResolver(DependencyResolver resolver) {
+ this.resolver = resolver;
+ }
+
+ /**
+ * Default constructor.
+ */
+ public TestDependencyResolver() {
+ this.resolver = new NoopDependencyResolver();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected <T> T doResolve(T instance) {
+ return resolver.resolve(instance);
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index 0280e9f..338d137 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -84,6 +84,7 @@ import org.apache.ignite.internal.binary.BinaryMarshaller;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.internal.processors.resource.DependencyResolver;
import org.apache.ignite.internal.processors.resource.GridSpringResourceContext;
import org.apache.ignite.internal.util.GridClassLoaderCache;
import org.apache.ignite.internal.util.GridTestClockTimer;
@@ -965,6 +966,25 @@ public abstract class GridAbstractTest extends JUnitAssertAware {
}
/**
+ * Starts new grid with given index and overriding {@link DependencyResolver}.
+ *
+ * @param idx Index of the grid to start.
+ * @param rslvr Dependency provider.
+ * @return Started grid.
+ * @throws Exception If anything failed.
+ */
+ protected IgniteEx startGrid(int idx, DependencyResolver rslvr) throws Exception {
+ IgnitionEx.dependencyResolver(rslvr);
+
+ try {
+ return startGrid(getTestIgniteInstanceName(idx));
+ }
+ finally {
+ IgnitionEx.dependencyResolver(null);
+ }
+ }
+
+ /**
* Starts new grid with given index.
*
* @param idx Index of the grid to start.
@@ -1036,6 +1056,25 @@ public abstract class GridAbstractTest extends JUnitAssertAware {
}
/**
+ * Starts new client grid with given index.
+ *
+ * @param idx Index of the grid to start.
+ * @param rslvr Dependency provider.
+ * @return Started grid.
+ * @throws Exception If anything failed.
+ */
+ protected IgniteEx startClientGrid(int idx, DependencyResolver rslvr) throws Exception {
+ IgnitionEx.dependencyResolver(rslvr);
+
+ try {
+ return startClientGrid(getTestIgniteInstanceName(idx));
+ }
+ finally {
+ IgnitionEx.dependencyResolver(null);
+ }
+ }
+
+ /**
* Starts new client grid with given name.
*
* @param igniteInstanceName Ignite instance name.
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/TxDeadlockDetectionTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/TxDeadlockDetectionTestSuite.java
index 255e07f..6814fc5 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/TxDeadlockDetectionTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/TxDeadlockDetectionTestSuite.java
@@ -22,6 +22,7 @@ import org.apache.ignite.internal.processors.cache.transactions.TxDeadlockDetect
import org.apache.ignite.internal.processors.cache.transactions.TxDeadlockDetectionNoHangsTest;
import org.apache.ignite.internal.processors.cache.transactions.TxDeadlockDetectionTest;
import org.apache.ignite.internal.processors.cache.transactions.TxDeadlockDetectionUnmasrhalErrorsTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxDeadlockOnEntryToStringTest;
import org.apache.ignite.internal.processors.cache.transactions.TxOptimisticDeadlockDetectionCrossCacheTest;
import org.apache.ignite.internal.processors.cache.transactions.TxOptimisticDeadlockDetectionTest;
import org.apache.ignite.internal.processors.cache.transactions.TxPessimisticDeadlockDetectionCrossCacheTest;
@@ -43,7 +44,8 @@ import org.junit.runners.Suite;
TxDeadlockDetectionTest.class,
TxDeadlockDetectionNoHangsTest.class,
TxDeadlockDetectionUnmasrhalErrorsTest.class,
- TxDeadlockDetectionMessageMarshallingTest.class
+ TxDeadlockDetectionMessageMarshallingTest.class,
+ TxDeadlockOnEntryToStringTest.class
})
public class TxDeadlockDetectionTestSuite {
}