You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@activemq.apache.org by cl...@apache.org on 2018/12/17 23:39:46 UTC
[1/3] activemq-artemis git commit: ARTEMIS-2197 Page deleted before
transaction finishes
Repository: activemq-artemis
Updated Branches:
refs/heads/2.6.x 877e16ccc -> 5bfc00cd0
ARTEMIS-2197 Page deleted before transaction finishes
Fixed regression caused by previous commit
b36dc37c152cabe3a0d9af178db043f842bfcdc0
(cherry picked from 2bd8fa7c5dcfb2c29a5c5f61ff433bf161b1b86e)
Project: http://git-wip-us.apache.org/repos/asf/activemq-artemis/repo
Commit: http://git-wip-us.apache.org/repos/asf/activemq-artemis/commit/8a2f65c7
Tree: http://git-wip-us.apache.org/repos/asf/activemq-artemis/tree/8a2f65c7
Diff: http://git-wip-us.apache.org/repos/asf/activemq-artemis/diff/8a2f65c7
Branch: refs/heads/2.6.x
Commit: 8a2f65c781b481001d6638d3c6cc4e8b70451067
Parents: b0a14ff
Author: Howard Gao <ho...@gmail.com>
Authored: Thu Dec 13 20:17:14 2018 +0800
Committer: Clebert Suconic <cl...@apache.org>
Committed: Mon Dec 17 18:39:38 2018 -0500
----------------------------------------------------------------------
.../artemis/core/server/impl/RefsOperation.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/8a2f65c7/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java
----------------------------------------------------------------------
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java
index cc31a04..3d8bdd6 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java
@@ -132,16 +132,16 @@ public class RefsOperation extends TransactionOperationAbstract {
message.incrementRefCount();
}
ackedTX.commit(true);
-
- if (pagedMessagesToPostACK != null) {
- for (MessageReference refmsg : pagedMessagesToPostACK) {
- ((PagedReference)refmsg).removePendingFlag();
- }
- }
} catch (Exception e) {
ActiveMQServerLogger.LOGGER.failedToProcessMessageReferenceAfterRollback(e);
}
}
+
+ if (pagedMessagesToPostACK != null) {
+ for (MessageReference refmsg : pagedMessagesToPostACK) {
+ ((PagedReference)refmsg).removePendingFlag();
+ }
+ }
}
protected void rollbackRedelivery(Transaction tx, MessageReference ref, long timeBase, Map<QueueImpl, LinkedList<MessageReference>> queueMap) throws Exception {
[3/3] activemq-artemis git commit: This closes #2462
Posted by cl...@apache.org.
This closes #2462
Project: http://git-wip-us.apache.org/repos/asf/activemq-artemis/repo
Commit: http://git-wip-us.apache.org/repos/asf/activemq-artemis/commit/5bfc00cd
Tree: http://git-wip-us.apache.org/repos/asf/activemq-artemis/tree/5bfc00cd
Diff: http://git-wip-us.apache.org/repos/asf/activemq-artemis/diff/5bfc00cd
Branch: refs/heads/2.6.x
Commit: 5bfc00cd011134b794e88686b20acbbd0e4eb44e
Parents: 877e16c 8a2f65c
Author: Clebert Suconic <cl...@apache.org>
Authored: Mon Dec 17 18:39:39 2018 -0500
Committer: Clebert Suconic <cl...@apache.org>
Committed: Mon Dec 17 18:39:39 2018 -0500
----------------------------------------------------------------------
.../core/paging/cursor/PageSubscription.java | 1 +
.../core/paging/cursor/PagedReference.java | 16 +++++++++
.../core/paging/cursor/PagedReferenceImpl.java | 10 ++++++
.../cursor/impl/PageSubscriptionImpl.java | 19 ++++++++--
.../artemis/core/server/impl/RefsOperation.java | 10 ++++++
.../integration/paging/PagingReceiveTest.java | 37 ++++++++++++++++++++
6 files changed, 90 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[2/3] activemq-artemis git commit: ARTEMIS-2197 Page deleted before
transaction finishes
Posted by cl...@apache.org.
ARTEMIS-2197 Page deleted before transaction finishes
When a receiving transaction is committed in a paging situation,
if a page happens to be completed and it will be deleted in a
transaction operation (PageCursorTx). The other tx operation
RefsOperation needs to access the page (in PageCache) to finish
its job. There is a chance that the PageCursorTx removes the
page before RefsOperation and it will cause the RefsOperation
failed to find a message in a page.
(cherry picked from b36dc37c152cabe3a0d9af178db043f842bfcdc0)
Project: http://git-wip-us.apache.org/repos/asf/activemq-artemis/repo
Commit: http://git-wip-us.apache.org/repos/asf/activemq-artemis/commit/b0a14ff3
Tree: http://git-wip-us.apache.org/repos/asf/activemq-artemis/tree/b0a14ff3
Diff: http://git-wip-us.apache.org/repos/asf/activemq-artemis/diff/b0a14ff3
Branch: refs/heads/2.6.x
Commit: b0a14ff3202f9aef6b29cb1f8764501469836d88
Parents: 877e16c
Author: Howard Gao <ho...@gmail.com>
Authored: Mon Dec 10 13:35:04 2018 +0800
Committer: Clebert Suconic <cl...@apache.org>
Committed: Mon Dec 17 18:39:38 2018 -0500
----------------------------------------------------------------------
.../core/paging/cursor/PageSubscription.java | 1 +
.../core/paging/cursor/PagedReference.java | 16 +++++++++
.../core/paging/cursor/PagedReferenceImpl.java | 10 ++++++
.../cursor/impl/PageSubscriptionImpl.java | 19 ++++++++--
.../artemis/core/server/impl/RefsOperation.java | 10 ++++++
.../integration/paging/PagingReceiveTest.java | 37 ++++++++++++++++++++
6 files changed, 90 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/b0a14ff3/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PageSubscription.java
----------------------------------------------------------------------
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PageSubscription.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PageSubscription.java
index b11362d..33092d5 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PageSubscription.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PageSubscription.java
@@ -170,4 +170,5 @@ public interface PageSubscription {
void incrementDeliveredSize(long size);
+ void removePendingDelivery(PagePosition position);
}
http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/b0a14ff3/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReference.java
----------------------------------------------------------------------
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReference.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReference.java
index be2d042..d9a7e59 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReference.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReference.java
@@ -28,4 +28,20 @@ public interface PagedReference extends MessageReference {
boolean isLargeMessage();
long getTransactionID();
+
+ /** this method affects paging clean up
+ It adds to the flag that prevents its page from cleanup.
+ It's a helper method to call the proper {@link PageSubscription#addPendingDelivery(PagePosition)}
+
+ @see PageSubscription#addPendingDelivery(PagePosition)
+ */
+ void addPendingFlag();
+
+ /** this method affects paging clean up
+ It adds to the flag that prevents its page from cleanup.
+ It's a helper method to call the proper {@link PageSubscription#addPendingDelivery(PagePosition)}
+
+ @see PageSubscription#addPendingDelivery(PagePosition)
+ */
+ void removePendingFlag();
}
http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/b0a14ff3/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReferenceImpl.java
----------------------------------------------------------------------
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReferenceImpl.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReferenceImpl.java
index 7bbabd1..0187105 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReferenceImpl.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/PagedReferenceImpl.java
@@ -324,6 +324,16 @@ public class PagedReferenceImpl extends LinkedListImpl.Node<PagedReferenceImpl>
}
@Override
+ public void addPendingFlag() {
+ subscription.addPendingDelivery(position);
+ }
+
+ @Override
+ public void removePendingFlag() {
+ subscription.removePendingDelivery(position);
+ }
+
+ @Override
public long getMessageID() {
if (messageID < 0) {
messageID = getPagedMessage().getMessage().getMessageID();
http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/b0a14ff3/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/impl/PageSubscriptionImpl.java
----------------------------------------------------------------------
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/impl/PageSubscriptionImpl.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/impl/PageSubscriptionImpl.java
index 5fec9a4..30e8528 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/impl/PageSubscriptionImpl.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/impl/PageSubscriptionImpl.java
@@ -59,7 +59,7 @@ import org.apache.activemq.artemis.utils.actors.ArtemisExecutor;
import org.apache.activemq.artemis.utils.collections.ConcurrentHashSet;
import org.jboss.logging.Logger;
-final class PageSubscriptionImpl implements PageSubscription {
+public final class PageSubscriptionImpl implements PageSubscription {
private static final Logger logger = Logger.getLogger(PageSubscriptionImpl.class);
@@ -556,6 +556,15 @@ final class PageSubscriptionImpl implements PageSubscription {
}
@Override
+ public void removePendingDelivery(final PagePosition position) {
+ PageCursorInfo info = getPageInfo(position);
+
+ if (info != null) {
+ info.decrementPendingTX();
+ }
+ }
+
+ @Override
public void redeliver(final PageIterator iterator, final PagePosition position) {
iterator.redeliver(position);
@@ -780,7 +789,7 @@ final class PageSubscriptionImpl implements PageSubscription {
return getPageInfo(pos.getPageNr());
}
- private PageCursorInfo getPageInfo(final long pageNr) {
+ public PageCursorInfo getPageInfo(final long pageNr) {
synchronized (consumedPages) {
PageCursorInfo pageInfo = consumedPages.get(pageNr);
@@ -916,7 +925,7 @@ final class PageSubscriptionImpl implements PageSubscription {
* This instance will be released as soon as the entire page is consumed, releasing the memory at
* that point The ref counts are increased also when a message is ignored for any reason.
*/
- private final class PageCursorInfo {
+ public final class PageCursorInfo {
// Number of messages existent on this page
private final int numberOfMessages;
@@ -934,6 +943,7 @@ final class PageSubscriptionImpl implements PageSubscription {
private final boolean wasLive;
// There's a pending TX to add elements on this page
+ // also can be used to prevent the page from being deleted too soon.
private final AtomicInteger pendingTX = new AtomicInteger(0);
// There's a pending delete on the async IO pipe
@@ -1108,6 +1118,9 @@ final class PageSubscriptionImpl implements PageSubscription {
return cache != null ? cache.get() : null;
}
+ public int getPendingTx() {
+ return pendingTX.get();
+ }
}
private final class PageCursorTX extends TransactionOperationAbstract {
http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/b0a14ff3/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java
----------------------------------------------------------------------
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java
index 9af8704..cc31a04 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/RefsOperation.java
@@ -67,6 +67,9 @@ public class RefsOperation extends TransactionOperationAbstract {
pagedMessagesToPostACK = new ArrayList<>();
}
pagedMessagesToPostACK.add(ref);
+ //here we do something to prevent page file
+ //from being deleted until the operation is done.
+ ((PagedReference)ref).addPendingFlag();
}
}
@@ -129,6 +132,12 @@ public class RefsOperation extends TransactionOperationAbstract {
message.incrementRefCount();
}
ackedTX.commit(true);
+
+ if (pagedMessagesToPostACK != null) {
+ for (MessageReference refmsg : pagedMessagesToPostACK) {
+ ((PagedReference)refmsg).removePendingFlag();
+ }
+ }
} catch (Exception e) {
ActiveMQServerLogger.LOGGER.failedToProcessMessageReferenceAfterRollback(e);
}
@@ -160,6 +169,7 @@ public class RefsOperation extends TransactionOperationAbstract {
if (pagedMessagesToPostACK != null) {
for (MessageReference refmsg : pagedMessagesToPostACK) {
+ ((PagedReference)refmsg).removePendingFlag();
if (((PagedReference) refmsg).isLargeMessage()) {
decrementRefCount(refmsg);
}
http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/b0a14ff3/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingReceiveTest.java
----------------------------------------------------------------------
diff --git a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingReceiveTest.java b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingReceiveTest.java
index 0223b42..347f3e4 100644
--- a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingReceiveTest.java
+++ b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingReceiveTest.java
@@ -23,6 +23,9 @@ import org.apache.activemq.artemis.api.core.client.ClientProducer;
import org.apache.activemq.artemis.api.core.client.ClientSession;
import org.apache.activemq.artemis.api.core.client.ClientSessionFactory;
import org.apache.activemq.artemis.api.core.client.ServerLocator;
+import org.apache.activemq.artemis.core.paging.PagingStore;
+import org.apache.activemq.artemis.core.paging.cursor.PageSubscription;
+import org.apache.activemq.artemis.core.paging.cursor.impl.PageSubscriptionImpl;
import org.apache.activemq.artemis.core.server.ActiveMQServer;
import org.apache.activemq.artemis.core.server.Queue;
import org.apache.activemq.artemis.api.core.RoutingType;
@@ -63,6 +66,40 @@ public class PagingReceiveTest extends ActiveMQTestBase {
assertEquals(numMsgs, queue.getMessagesAdded());
}
+ @Test
+ public void testReceiveTx() throws Exception {
+ receiveAllMessagesTxAndPageCheckPendingTx();
+ }
+
+ private void receiveAllMessagesTxAndPageCheckPendingTx() throws Exception {
+ final ClientSessionFactory sf = createSessionFactory(locator);
+ ClientSession session = sf.createSession(null, null, false, true, false, false, 0);
+
+ session.start();
+ ClientConsumer consumer = session.createConsumer(ADDRESS);
+ for (int i = 0; i < numMsgs; i++) {
+ ClientMessage message = consumer.receive(2000);
+ assertNotNull(message);
+ message.acknowledge();
+ }
+
+ //before committing the pendingTx should be positive.
+ PagingStore store = server.getPagingManager().getPageStore(ADDRESS);
+ long qid = server.locateQueue(ADDRESS).getID();
+ PageSubscription pageSub = store.getCursorProvider().getSubscription(qid);
+ long pageNr = store.getCurrentWritingPage();
+
+ PageSubscriptionImpl.PageCursorInfo info = ((PageSubscriptionImpl)pageSub).getPageInfo(pageNr);
+
+ System.out.println("pendingTx: " + info.getPendingTx());
+
+ //The positive pendingTx will prevent a page being removed
+ //before ResOperation is completed.
+ assertTrue(info.getPendingTx() > 0);
+
+ session.commit();
+ session.close();
+ }
@Override
@Before