You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dg...@apache.org on 2018/09/19 16:06:06 UTC
ignite git commit: IGNITE-8855 Throttle frequently reconnect client -
Fixes #4739.
Repository: ignite
Updated Branches:
refs/heads/master 1efec196e -> 58150f8fe
IGNITE-8855 Throttle frequently reconnect client - Fixes #4739.
Signed-off-by: Dmitriy Govorukhin <dm...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/58150f8f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/58150f8f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/58150f8f
Branch: refs/heads/master
Commit: 58150f8fed16255a48b2d110ce81afd02189d73b
Parents: 1efec19
Author: ibessonov <be...@gmail.com>
Authored: Wed Sep 19 19:05:10 2018 +0300
Committer: Dmitriy Govorukhin <dm...@gmail.com>
Committed: Wed Sep 19 19:05:10 2018 +0300
----------------------------------------------------------------------
.../apache/ignite/IgniteSystemProperties.java | 4 ++
.../ignite/spi/discovery/tcp/ClientImpl.java | 47 ++++++++++++++++++++
2 files changed, 51 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/58150f8f/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 9e21814..1db7296 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -506,6 +506,10 @@ public final class IgniteSystemProperties {
public static final String IGNITE_DISCOVERY_CLIENT_RECONNECT_HISTORY_SIZE =
"IGNITE_DISCOVERY_CLIENT_RECONNECT_HISTORY_SIZE";
+ /** Time interval that indicates that client reconnect throttle must be reset to zero. 2 minutes by default. */
+ public static final String CLIENT_THROTTLE_RECONNECT_RESET_TIMEOUT_INTERVAL =
+ "CLIENT_THROTTLE_RECONNECT_RESET_TIMEOUT_INTERVAL";
+
/** Number of cache operation retries in case of topology exceptions. */
public static final String IGNITE_CACHE_RETRIES_COUNT = "IGNITE_CACHE_RETRIES_COUNT";
http://git-wip-us.apache.org/repos/asf/ignite/blob/58150f8f/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 312f737..673290e 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -48,6 +48,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicReference;
import javax.net.ssl.SSLException;
import org.apache.ignite.Ignite;
@@ -60,6 +61,7 @@ import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.cache.CacheMetrics;
import org.apache.ignite.cluster.ClusterMetrics;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.failure.FailureContext;
import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
import org.apache.ignite.internal.IgniteEx;
@@ -79,6 +81,7 @@ import org.apache.ignite.internal.util.worker.GridWorker;
import org.apache.ignite.internal.worker.WorkersRegistry;
import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.spi.IgniteSpiAdapter;
import org.apache.ignite.spi.IgniteSpiContext;
import org.apache.ignite.spi.IgniteSpiException;
import org.apache.ignite.spi.IgniteSpiOperationTimeoutHelper;
@@ -147,6 +150,12 @@ class ClientImpl extends TcpDiscoveryImpl {
/** */
private static final Object SPI_RECONNECT = "SPI_RECONNECT";
+ /** */
+ private static final long CLIENT_THROTTLE_RECONNECT_RESET_TIMEOUT = IgniteSystemProperties.getLong(
+ IgniteSystemProperties.CLIENT_THROTTLE_RECONNECT_RESET_TIMEOUT_INTERVAL,
+ 2 * 60_000
+ );
+
/** Remote nodes. */
private final ConcurrentMap<UUID, TcpDiscoveryNode> rmtNodes = new ConcurrentHashMap<>();
@@ -1615,6 +1624,12 @@ class ClientImpl extends TcpDiscoveryImpl {
/** */
private boolean nodeAdded;
+ /** */
+ private long lastReconnectTimestamp = -1;
+
+ /** */
+ private long currentReconnectDelay = -1;
+
/**
* @param log Logger.
*/
@@ -1697,6 +1712,8 @@ class ClientImpl extends TcpDiscoveryImpl {
locNode.onClientDisconnected(newId);
+ throttleClientReconnect();
+
tryJoin();
}
}
@@ -1886,6 +1903,36 @@ class ClientImpl extends TcpDiscoveryImpl {
}
/**
+ * Wait random delay before trying to reconnect. Delay will grow exponentially every time client is forced to
+ * reconnect, but only if all these reconnections happened in small period of time (2 minutes). Maximum delay
+ * could be configured with {@link IgniteSpiAdapter#clientFailureDetectionTimeout()}, default value is
+ * {@link IgniteConfiguration#DFLT_CLIENT_FAILURE_DETECTION_TIMEOUT}.
+ *
+ * @throws InterruptedException If thread is interrupted.
+ */
+ private void throttleClientReconnect() throws InterruptedException {
+ if (U.currentTimeMillis() - lastReconnectTimestamp > CLIENT_THROTTLE_RECONNECT_RESET_TIMEOUT)
+ currentReconnectDelay = 0; // Skip pause on first reconnect.
+ else if (currentReconnectDelay == 0)
+ currentReconnectDelay = 200;
+ else {
+ long maxDelay = spi.failureDetectionTimeoutEnabled()
+ ? spi.clientFailureDetectionTimeout()
+ : IgniteConfiguration.DFLT_CLIENT_FAILURE_DETECTION_TIMEOUT;
+
+ currentReconnectDelay = Math.min(maxDelay, (int)(currentReconnectDelay * 1.5));
+ }
+
+ if (currentReconnectDelay != 0) {
+ ThreadLocalRandom random = ThreadLocalRandom.current();
+
+ Thread.sleep(random.nextLong(currentReconnectDelay / 2, currentReconnectDelay));
+ }
+
+ lastReconnectTimestamp = U.currentTimeMillis();
+ }
+
+ /**
*
*/
private void onDisconnected() {