You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@activemq.apache.org by cl...@apache.org on 2021/08/06 12:34:38 UTC

[activemq-artemis] 01/03: ARTEMIS-3275 Lock CORE client communication during failover retries

This is an automated email from the ASF dual-hosted git repository.

clebertsuconic pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/activemq-artemis.git

commit 6d2b96c79e75fb230ec0fd9f34d7910d7644593f
Author: Domenico Francesco Bruscino <br...@apache.org>
AuthorDate: Tue Jun 22 09:52:48 2021 +0200

    ARTEMIS-3275 Lock CORE client communication during failover retries
---
 .../core/client/impl/ClientSessionFactoryImpl.java | 38 +++++++++++-------
 .../core/client/impl/ClientSessionImpl.java        | 46 ++++++++++++++--------
 .../core/client/impl/ClientSessionInternal.java    |  2 +
 .../artemis/core/protocol/core/Channel.java        |  7 ++++
 .../protocol/core/impl/ActiveMQSessionContext.java |  2 +-
 .../core/protocol/core/impl/ChannelImpl.java       |  5 +++
 .../integration/cluster/failover/FailoverTest.java | 41 +++++++++++++++++++
 .../integration/cluster/util/BackupSyncDelay.java  |  5 +++
 8 files changed, 114 insertions(+), 32 deletions(-)

diff --git a/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/ClientSessionFactoryImpl.java b/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/ClientSessionFactoryImpl.java
index 40fb156..92236f4 100644
--- a/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/ClientSessionFactoryImpl.java
+++ b/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/ClientSessionFactoryImpl.java
@@ -632,10 +632,19 @@ public class ClientSessionFactoryImpl implements ClientSessionFactoryInternal, C
 
                connector = null;
 
-               boolean allSessionReconnected;
+               HashSet<ClientSessionInternal> sessionsToFailover;
+               synchronized (sessions) {
+                  sessionsToFailover = new HashSet<>(sessions);
+               }
+
+               for (ClientSessionInternal session : sessionsToFailover) {
+                  session.preHandleFailover(connection);
+               }
+
+               boolean allSessionReconnected = false;
                int failedReconnectSessionsCounter = 0;
                do {
-                  allSessionReconnected = reconnectSessions(oldConnection, reconnectAttempts, me);
+                  allSessionReconnected = reconnectSessions(sessionsToFailover, oldConnection, reconnectAttempts, me);
                   if (oldConnection != null) {
                      oldConnection.destroy();
                   }
@@ -644,10 +653,19 @@ public class ClientSessionFactoryImpl implements ClientSessionFactoryInternal, C
                      failedReconnectSessionsCounter++;
                      oldConnection = connection;
                      connection = null;
+
+                     // Wait for retry when the connection is established but not all session are reconnected.
+                     if ((reconnectAttempts == -1 || failedReconnectSessionsCounter < reconnectAttempts) && oldConnection != null) {
+                        waitForRetry(retryInterval);
+                     }
                   }
                }
                while ((reconnectAttempts == -1 || failedReconnectSessionsCounter < reconnectAttempts) && !allSessionReconnected);
 
+               for (ClientSessionInternal session : sessionsToFailover) {
+                  session.postHandleFailover(connection, allSessionReconnected);
+               }
+
                if (oldConnection != null) {
                   oldConnection.destroy();
                }
@@ -764,18 +782,10 @@ public class ClientSessionFactoryImpl implements ClientSessionFactoryInternal, C
    /*
     * Re-attach sessions all pre-existing sessions to the new remoting connection
     */
-   private boolean reconnectSessions(final RemotingConnection oldConnection,
-                                  final int reconnectAttempts,
-                                  final ActiveMQException cause) {
-      HashSet<ClientSessionInternal> sessionsToFailover;
-      synchronized (sessions) {
-         sessionsToFailover = new HashSet<>(sessions);
-      }
-
-      for (ClientSessionInternal session : sessionsToFailover) {
-         session.preHandleFailover(connection);
-      }
-
+   private boolean reconnectSessions(final Set<ClientSessionInternal> sessionsToFailover,
+                                     final RemotingConnection oldConnection,
+                                     final int reconnectAttempts,
+                                     final ActiveMQException cause) {
       getConnectionWithRetry(reconnectAttempts, oldConnection);
 
       if (connection == null) {
diff --git a/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/ClientSessionImpl.java b/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/ClientSessionImpl.java
index d3a66a5..33e5a77 100644
--- a/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/ClientSessionImpl.java
+++ b/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/ClientSessionImpl.java
@@ -141,6 +141,8 @@ public final class ClientSessionImpl implements ClientSessionInternal, FailureLi
 
    private volatile boolean mayAttemptToFailover = true;
 
+   private volatile boolean resetCreditManager = false;
+
    /**
     * Current XID. this will be used in case of failover
     */
@@ -1387,8 +1389,6 @@ public final class ClientSessionImpl implements ClientSessionInternal, FailureLi
             return true;
          }
 
-         boolean resetCreditManager = false;
-
          try {
 
             // TODO remove this and encapsulate it
@@ -1463,30 +1463,42 @@ public final class ClientSessionImpl implements ClientSessionInternal, FailureLi
          } catch (Throwable t) {
             ActiveMQClientLogger.LOGGER.failedToHandleFailover(t);
             suc = false;
-         } finally {
-            sessionContext.releaseCommunications();
          }
+      }
+
+      return suc;
+   }
 
-         if (resetCreditManager) {
-            synchronized (producerCreditManager) {
-               producerCreditManager.reset();
+   @Override
+   public void postHandleFailover(RemotingConnection connection, boolean successful) {
+      sessionContext.releaseCommunications();
+
+      if (successful) {
+         synchronized (this) {
+            if (closed) {
+               return;
             }
 
-            // Also need to send more credits for consumers, otherwise the system could hand with the server
-            // not having any credits to send
-         }
-      }
+            if (resetCreditManager) {
+               synchronized (producerCreditManager) {
+                  producerCreditManager.reset();
+               }
 
-      HashMap<String, String> metaDataToSend;
+               resetCreditManager = false;
 
-      synchronized (metadata) {
-         metaDataToSend = new HashMap<>(metadata);
-      }
+               // Also need to send more credits for consumers, otherwise the system could hand with the server
+               // not having any credits to send
+            }
+         }
 
-      sessionContext.resetMetadata(metaDataToSend);
+         HashMap<String, String> metaDataToSend;
 
-      return suc;
+         synchronized (metadata) {
+            metaDataToSend = new HashMap<>(metadata);
+         }
 
+         sessionContext.resetMetadata(metaDataToSend);
+      }
    }
 
    @Override
diff --git a/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/ClientSessionInternal.java b/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/ClientSessionInternal.java
index a3700b2..173087d 100644
--- a/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/ClientSessionInternal.java
+++ b/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/ClientSessionInternal.java
@@ -68,6 +68,8 @@ public interface ClientSessionInternal extends ClientSession {
 
    boolean handleFailover(RemotingConnection backupConnection, ActiveMQException cause);
 
+   void postHandleFailover(RemotingConnection connection, boolean successful);
+
    RemotingConnection getConnection();
 
    void cleanUp(boolean failingOver) throws ActiveMQException;
diff --git a/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/protocol/core/Channel.java b/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/protocol/core/Channel.java
index 372cad4..1281729 100644
--- a/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/protocol/core/Channel.java
+++ b/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/protocol/core/Channel.java
@@ -185,6 +185,13 @@ public interface Channel {
    int getLastConfirmedCommandID();
 
    /**
+    * queries if this channel is locked. This method is designed for use in monitoring of the system state, not for synchronization control.
+    *
+    * @return true it the channel is locked and false otherwise
+    */
+   boolean isLocked();
+
+   /**
     * locks the channel.
     * <p>
     * While locked no packets can be sent or received
diff --git a/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/protocol/core/impl/ActiveMQSessionContext.java b/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/protocol/core/impl/ActiveMQSessionContext.java
index 4e122a9..811ebef 100644
--- a/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/protocol/core/impl/ActiveMQSessionContext.java
+++ b/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/protocol/core/impl/ActiveMQSessionContext.java
@@ -139,7 +139,7 @@ public class ActiveMQSessionContext extends SessionContext {
    private String name;
    private boolean killed;
 
-   protected Channel getSessionChannel() {
+   public Channel getSessionChannel() {
       return sessionChannel;
    }
 
diff --git a/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/protocol/core/impl/ChannelImpl.java b/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/protocol/core/impl/ChannelImpl.java
index 4ecd2c6..18bb08c 100644
--- a/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/protocol/core/impl/ChannelImpl.java
+++ b/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/protocol/core/impl/ChannelImpl.java
@@ -690,6 +690,11 @@ public final class ChannelImpl implements Channel {
    }
 
    @Override
+   public boolean isLocked() {
+      return failingOver;
+   }
+
+   @Override
    public void lock() {
       if (logger.isTraceEnabled()) {
          logger.trace("RemotingConnectionID=" + (connection == null ? "NULL" : connection.getID()) + " lock channel " + this);
diff --git a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/failover/FailoverTest.java b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/failover/FailoverTest.java
index 3dcf9a9..43756f1 100644
--- a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/failover/FailoverTest.java
+++ b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/failover/FailoverTest.java
@@ -27,6 +27,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.activemq.artemis.api.core.ActiveMQDuplicateIdException;
 import org.apache.activemq.artemis.api.core.ActiveMQException;
@@ -50,8 +51,10 @@ import org.apache.activemq.artemis.api.core.client.ServerLocator;
 import org.apache.activemq.artemis.api.core.client.SessionFailureListener;
 import org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl;
 import org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryInternal;
+import org.apache.activemq.artemis.core.client.impl.ClientSessionInternal;
 import org.apache.activemq.artemis.core.protocol.core.Channel;
 import org.apache.activemq.artemis.core.protocol.core.Packet;
+import org.apache.activemq.artemis.core.protocol.core.impl.ActiveMQSessionContext;
 import org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl;
 import org.apache.activemq.artemis.core.protocol.core.impl.PacketImpl;
 import org.apache.activemq.artemis.core.protocol.core.impl.RemotingConnectionImpl;
@@ -1972,6 +1975,44 @@ public class FailoverTest extends FailoverTestBase {
    }
 
    @Test(timeout = 120000)
+   public void testChannelStateDuringFailover() throws Exception {
+      locator.setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true).setReconnectAttempts(300).setRetryInterval(100);
+
+      sf = createSessionFactoryAndWaitForTopology(locator, 2);
+
+      final AtomicBoolean channelLockedDuringFailover = new AtomicBoolean(false);
+
+      ClientSession session = createSession(sf, true, true, 0);
+
+      backupServer.addInterceptor(
+         new Interceptor() {
+            private int index = 0;
+
+            @Override
+            public boolean intercept(Packet packet, RemotingConnection connection) throws ActiveMQException {
+               if (index < 1 && packet.getType() == PacketImpl.CREATESESSION) {
+                  sf.getConnection().addCloseListener(() -> {
+                     index++;
+                     ActiveMQSessionContext sessionContext = (ActiveMQSessionContext)((ClientSessionInternal)session).getSessionContext();
+                     channelLockedDuringFailover.set(sessionContext.getSessionChannel().isLocked());
+                  });
+
+                  Channel sessionChannel = ((RemotingConnectionImpl)connection).getChannel(ChannelImpl.CHANNEL_ID.SESSION.id, -1);
+                  sessionChannel.send(new ActiveMQExceptionMessage(new ActiveMQInternalErrorException()));
+                  return false;
+               }
+               return true;
+            }
+         });
+
+      session.start();
+
+      crash(session);
+
+      Assert.assertTrue(channelLockedDuringFailover.get());
+   }
+
+   @Test(timeout = 120000)
    public void testForceBlockingReturn() throws Exception {
       locator.setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true).setReconnectAttempts(300).setRetryInterval(100);
 
diff --git a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/util/BackupSyncDelay.java b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/util/BackupSyncDelay.java
index b0af71b..287d730 100644
--- a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/util/BackupSyncDelay.java
+++ b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/util/BackupSyncDelay.java
@@ -301,6 +301,11 @@ public class BackupSyncDelay implements Interceptor {
       }
 
       @Override
+      public boolean isLocked() {
+         return false;
+      }
+
+      @Override
       public void lock() {
          throw new UnsupportedOperationException();
       }