You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@activemq.apache.org by cl...@apache.org on 2018/12/17 23:39:46 UTC

[1/3] activemq-artemis git commit: ARTEMIS-2197 Page deleted before transaction finishes

Repository: activemq-artemis
Updated Branches:
  refs/heads/2.6.x 877e16ccc -> 5bfc00cd0


ARTEMIS-2197 Page deleted before transaction finishes

Fixed regression caused by previous commit
b36dc37c152cabe3a0d9af178db043f842bfcdc0

(cherry picked from 2bd8fa7c5dcfb2c29a5c5f61ff433bf161b1b86e)


Project: http://git-wip-us.apache.org/repos/asf/activemq-artemis/repo
Commit: http://git-wip-us.apache.org/repos/asf/activemq-artemis/commit/8a2f65c7
Tree: http://git-wip-us.apache.org/repos/asf/activemq-artemis/tree/8a2f65c7
Diff: http://git-wip-us.apache.org/repos/asf/activemq-artemis/diff/8a2f65c7

Branch: refs/heads/2.6.x
Commit: 8a2f65c781b481001d6638d3c6cc4e8b70451067
Parents: b0a14ff
Author: Howard Gao <ho...@gmail.com>
Authored: Thu Dec 13 20:17:14 2018 +0800
Committer: Clebert Suconic <cl...@apache.org>
Committed: Mon Dec 17 18:39:38 2018 -0500

----------------------------------------------------------------------
 .../artemis/core/server/impl/RefsOperation.java         | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/8a2f65c7/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java
----------------------------------------------------------------------
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java
index cc31a04..3d8bdd6 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java
@@ -132,16 +132,16 @@ public class RefsOperation extends TransactionOperationAbstract {
                message.incrementRefCount();
             }
             ackedTX.commit(true);
-
-            if (pagedMessagesToPostACK != null) {
-               for (MessageReference refmsg : pagedMessagesToPostACK) {
-                  ((PagedReference)refmsg).removePendingFlag();
-               }
-            }
          } catch (Exception e) {
             ActiveMQServerLogger.LOGGER.failedToProcessMessageReferenceAfterRollback(e);
          }
       }
+
+      if (pagedMessagesToPostACK != null) {
+         for (MessageReference refmsg : pagedMessagesToPostACK) {
+            ((PagedReference)refmsg).removePendingFlag();
+         }
+      }
    }
 
    protected void rollbackRedelivery(Transaction tx, MessageReference ref, long timeBase, Map<QueueImpl, LinkedList<MessageReference>> queueMap) throws Exception {


[3/3] activemq-artemis git commit: This closes #2462

Posted by cl...@apache.org.
This closes #2462


Project: http://git-wip-us.apache.org/repos/asf/activemq-artemis/repo
Commit: http://git-wip-us.apache.org/repos/asf/activemq-artemis/commit/5bfc00cd
Tree: http://git-wip-us.apache.org/repos/asf/activemq-artemis/tree/5bfc00cd
Diff: http://git-wip-us.apache.org/repos/asf/activemq-artemis/diff/5bfc00cd

Branch: refs/heads/2.6.x
Commit: 5bfc00cd011134b794e88686b20acbbd0e4eb44e
Parents: 877e16c 8a2f65c
Author: Clebert Suconic <cl...@apache.org>
Authored: Mon Dec 17 18:39:39 2018 -0500
Committer: Clebert Suconic <cl...@apache.org>
Committed: Mon Dec 17 18:39:39 2018 -0500

----------------------------------------------------------------------
 .../core/paging/cursor/PageSubscription.java    |  1 +
 .../core/paging/cursor/PagedReference.java      | 16 +++++++++
 .../core/paging/cursor/PagedReferenceImpl.java  | 10 ++++++
 .../cursor/impl/PageSubscriptionImpl.java       | 19 ++++++++--
 .../artemis/core/server/impl/RefsOperation.java | 10 ++++++
 .../integration/paging/PagingReceiveTest.java   | 37 ++++++++++++++++++++
 6 files changed, 90 insertions(+), 3 deletions(-)
----------------------------------------------------------------------



[2/3] activemq-artemis git commit: ARTEMIS-2197 Page deleted before transaction finishes

Posted by cl...@apache.org.
ARTEMIS-2197 Page deleted before transaction finishes

When a receiving transaction is committed in a paging situation,
if a page happens to be completed and it will be deleted in a
transaction operation (PageCursorTx). The other tx operation
RefsOperation needs to access the page (in PageCache) to finish
its job. There is a chance that the PageCursorTx removes the
page before RefsOperation and it will cause the RefsOperation
failed to find a message in a page.

(cherry picked from b36dc37c152cabe3a0d9af178db043f842bfcdc0)


Project: http://git-wip-us.apache.org/repos/asf/activemq-artemis/repo
Commit: http://git-wip-us.apache.org/repos/asf/activemq-artemis/commit/b0a14ff3
Tree: http://git-wip-us.apache.org/repos/asf/activemq-artemis/tree/b0a14ff3
Diff: http://git-wip-us.apache.org/repos/asf/activemq-artemis/diff/b0a14ff3

Branch: refs/heads/2.6.x
Commit: b0a14ff3202f9aef6b29cb1f8764501469836d88
Parents: 877e16c
Author: Howard Gao <ho...@gmail.com>
Authored: Mon Dec 10 13:35:04 2018 +0800
Committer: Clebert Suconic <cl...@apache.org>
Committed: Mon Dec 17 18:39:38 2018 -0500

----------------------------------------------------------------------
 .../core/paging/cursor/PageSubscription.java    |  1 +
 .../core/paging/cursor/PagedReference.java      | 16 +++++++++
 .../core/paging/cursor/PagedReferenceImpl.java  | 10 ++++++
 .../cursor/impl/PageSubscriptionImpl.java       | 19 ++++++++--
 .../artemis/core/server/impl/RefsOperation.java | 10 ++++++
 .../integration/paging/PagingReceiveTest.java   | 37 ++++++++++++++++++++
 6 files changed, 90 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/b0a14ff3/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PageSubscription.java
----------------------------------------------------------------------
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PageSubscription.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PageSubscription.java
index b11362d..33092d5 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PageSubscription.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PageSubscription.java
@@ -170,4 +170,5 @@ public interface PageSubscription {
 
    void incrementDeliveredSize(long size);
 
+   void removePendingDelivery(PagePosition position);
 }

http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/b0a14ff3/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReference.java
----------------------------------------------------------------------
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReference.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReference.java
index be2d042..d9a7e59 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReference.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReference.java
@@ -28,4 +28,20 @@ public interface PagedReference extends MessageReference {
    boolean isLargeMessage();
 
    long getTransactionID();
+
+   /** this method affects paging clean up
+       It adds to the flag that prevents its page from cleanup.
+       It's a helper method to call the proper {@link PageSubscription#addPendingDelivery(PagePosition)}
+
+    @see PageSubscription#addPendingDelivery(PagePosition)
+    */
+   void addPendingFlag();
+
+   /** this method affects paging clean up
+    It adds to the flag that prevents its page from cleanup.
+    It's a helper method to call the proper {@link PageSubscription#addPendingDelivery(PagePosition)}
+
+    @see PageSubscription#addPendingDelivery(PagePosition)
+    */
+   void removePendingFlag();
 }

http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/b0a14ff3/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReferenceImpl.java
----------------------------------------------------------------------
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReferenceImpl.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReferenceImpl.java
index 7bbabd1..0187105 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReferenceImpl.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReferenceImpl.java
@@ -324,6 +324,16 @@ public class PagedReferenceImpl extends LinkedListImpl.Node<PagedReferenceImpl>
    }
 
    @Override
+   public void addPendingFlag() {
+      subscription.addPendingDelivery(position);
+   }
+
+   @Override
+   public void removePendingFlag() {
+      subscription.removePendingDelivery(position);
+   }
+
+   @Override
    public long getMessageID() {
       if (messageID < 0) {
          messageID = getPagedMessage().getMessage().getMessageID();

http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/b0a14ff3/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/impl/PageSubscriptionImpl.java
----------------------------------------------------------------------
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/impl/PageSubscriptionImpl.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/impl/PageSubscriptionImpl.java
index 5fec9a4..30e8528 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/impl/PageSubscriptionImpl.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/impl/PageSubscriptionImpl.java
@@ -59,7 +59,7 @@ import org.apache.activemq.artemis.utils.actors.ArtemisExecutor;
 import org.apache.activemq.artemis.utils.collections.ConcurrentHashSet;
 import org.jboss.logging.Logger;
 
-final class PageSubscriptionImpl implements PageSubscription {
+public final class PageSubscriptionImpl implements PageSubscription {
 
    private static final Logger logger = Logger.getLogger(PageSubscriptionImpl.class);
 
@@ -556,6 +556,15 @@ final class PageSubscriptionImpl implements PageSubscription {
    }
 
    @Override
+   public void removePendingDelivery(final PagePosition position) {
+      PageCursorInfo info = getPageInfo(position);
+
+      if (info != null) {
+         info.decrementPendingTX();
+      }
+   }
+
+   @Override
    public void redeliver(final PageIterator iterator, final PagePosition position) {
       iterator.redeliver(position);
 
@@ -780,7 +789,7 @@ final class PageSubscriptionImpl implements PageSubscription {
       return getPageInfo(pos.getPageNr());
    }
 
-   private PageCursorInfo getPageInfo(final long pageNr) {
+   public PageCursorInfo getPageInfo(final long pageNr) {
       synchronized (consumedPages) {
          PageCursorInfo pageInfo = consumedPages.get(pageNr);
 
@@ -916,7 +925,7 @@ final class PageSubscriptionImpl implements PageSubscription {
     * This instance will be released as soon as the entire page is consumed, releasing the memory at
     * that point The ref counts are increased also when a message is ignored for any reason.
     */
-   private final class PageCursorInfo {
+   public final class PageCursorInfo {
 
       // Number of messages existent on this page
       private final int numberOfMessages;
@@ -934,6 +943,7 @@ final class PageSubscriptionImpl implements PageSubscription {
       private final boolean wasLive;
 
       // There's a pending TX to add elements on this page
+      // also can be used to prevent the page from being deleted too soon.
       private final AtomicInteger pendingTX = new AtomicInteger(0);
 
       // There's a pending delete on the async IO pipe
@@ -1108,6 +1118,9 @@ final class PageSubscriptionImpl implements PageSubscription {
          return cache != null ? cache.get() : null;
       }
 
+      public int getPendingTx() {
+         return pendingTX.get();
+      }
    }
 
    private final class PageCursorTX extends TransactionOperationAbstract {

http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/b0a14ff3/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java
----------------------------------------------------------------------
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java
index 9af8704..cc31a04 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java
@@ -67,6 +67,9 @@ public class RefsOperation extends TransactionOperationAbstract {
             pagedMessagesToPostACK = new ArrayList<>();
          }
          pagedMessagesToPostACK.add(ref);
+         //here we do something to prevent page file
+         //from being deleted until the operation is done.
+         ((PagedReference)ref).addPendingFlag();
       }
    }
 
@@ -129,6 +132,12 @@ public class RefsOperation extends TransactionOperationAbstract {
                message.incrementRefCount();
             }
             ackedTX.commit(true);
+
+            if (pagedMessagesToPostACK != null) {
+               for (MessageReference refmsg : pagedMessagesToPostACK) {
+                  ((PagedReference)refmsg).removePendingFlag();
+               }
+            }
          } catch (Exception e) {
             ActiveMQServerLogger.LOGGER.failedToProcessMessageReferenceAfterRollback(e);
          }
@@ -160,6 +169,7 @@ public class RefsOperation extends TransactionOperationAbstract {
 
       if (pagedMessagesToPostACK != null) {
          for (MessageReference refmsg : pagedMessagesToPostACK) {
+            ((PagedReference)refmsg).removePendingFlag();
             if (((PagedReference) refmsg).isLargeMessage()) {
                decrementRefCount(refmsg);
             }

http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/b0a14ff3/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingReceiveTest.java
----------------------------------------------------------------------
diff --git a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingReceiveTest.java b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingReceiveTest.java
index 0223b42..347f3e4 100644
--- a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingReceiveTest.java
+++ b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingReceiveTest.java
@@ -23,6 +23,9 @@ import org.apache.activemq.artemis.api.core.client.ClientProducer;
 import org.apache.activemq.artemis.api.core.client.ClientSession;
 import org.apache.activemq.artemis.api.core.client.ClientSessionFactory;
 import org.apache.activemq.artemis.api.core.client.ServerLocator;
+import org.apache.activemq.artemis.core.paging.PagingStore;
+import org.apache.activemq.artemis.core.paging.cursor.PageSubscription;
+import org.apache.activemq.artemis.core.paging.cursor.impl.PageSubscriptionImpl;
 import org.apache.activemq.artemis.core.server.ActiveMQServer;
 import org.apache.activemq.artemis.core.server.Queue;
 import org.apache.activemq.artemis.api.core.RoutingType;
@@ -63,6 +66,40 @@ public class PagingReceiveTest extends ActiveMQTestBase {
       assertEquals(numMsgs, queue.getMessagesAdded());
    }
 
+   @Test
+   public void testReceiveTx() throws Exception {
+      receiveAllMessagesTxAndPageCheckPendingTx();
+   }
+
+   private void receiveAllMessagesTxAndPageCheckPendingTx() throws Exception {
+      final ClientSessionFactory sf = createSessionFactory(locator);
+      ClientSession session = sf.createSession(null, null, false, true, false, false, 0);
+
+      session.start();
+      ClientConsumer consumer = session.createConsumer(ADDRESS);
+      for (int i = 0; i < numMsgs; i++) {
+         ClientMessage message = consumer.receive(2000);
+         assertNotNull(message);
+         message.acknowledge();
+      }
+
+      //before committing the pendingTx should be positive.
+      PagingStore store = server.getPagingManager().getPageStore(ADDRESS);
+      long qid = server.locateQueue(ADDRESS).getID();
+      PageSubscription pageSub = store.getCursorProvider().getSubscription(qid);
+      long pageNr = store.getCurrentWritingPage();
+
+      PageSubscriptionImpl.PageCursorInfo info = ((PageSubscriptionImpl)pageSub).getPageInfo(pageNr);
+
+      System.out.println("pendingTx: " + info.getPendingTx());
+
+      //The positive pendingTx will prevent a page being removed
+      //before ResOperation is completed.
+      assertTrue(info.getPendingTx() > 0);
+
+      session.commit();
+      session.close();
+   }
 
    @Override
    @Before