[3/7] activemq-artemis git commit: ARTEMIS-581 Implement max disk usage, and global-max-size

[ma...@apache.org]

http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/4472aa0e/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingTest.java
----------------------------------------------------------------------
diff --git a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingTest.java b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingTest.java
new file mode 100644
index 0000000..82ce14f
--- /dev/null
+++ b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingTest.java
@@ -0,0 +1,5731 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.activemq.artemis.tests.integration.paging;
+
+import javax.transaction.xa.XAResource;
+import javax.transaction.xa.Xid;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.lang.management.ManagementFactory;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.activemq.artemis.api.core.ActiveMQBuffer;
+import org.apache.activemq.artemis.api.core.ActiveMQException;
+import org.apache.activemq.artemis.api.core.ActiveMQExceptionType;
+import org.apache.activemq.artemis.api.core.Message;
+import org.apache.activemq.artemis.api.core.Pair;
+import org.apache.activemq.artemis.api.core.SimpleString;
+import org.apache.activemq.artemis.api.core.client.ClientConsumer;
+import org.apache.activemq.artemis.api.core.client.ClientMessage;
+import org.apache.activemq.artemis.api.core.client.ClientProducer;
+import org.apache.activemq.artemis.api.core.client.ClientSession;
+import org.apache.activemq.artemis.api.core.client.ClientSessionFactory;
+import org.apache.activemq.artemis.api.core.client.MessageHandler;
+import org.apache.activemq.artemis.api.core.client.ServerLocator;
+import org.apache.activemq.artemis.core.client.impl.ClientConsumerInternal;
+import org.apache.activemq.artemis.core.config.Configuration;
+import org.apache.activemq.artemis.core.config.DivertConfiguration;
+import org.apache.activemq.artemis.core.filter.Filter;
+import org.apache.activemq.artemis.core.io.IOCallback;
+import org.apache.activemq.artemis.core.io.nio.NIOSequentialFileFactory;
+import org.apache.activemq.artemis.core.journal.PreparedTransactionInfo;
+import org.apache.activemq.artemis.core.journal.RecordInfo;
+import org.apache.activemq.artemis.core.journal.impl.JournalImpl;
+import org.apache.activemq.artemis.core.paging.PagingManager;
+import org.apache.activemq.artemis.core.paging.PagingStore;
+import org.apache.activemq.artemis.core.paging.cursor.PageCursorProvider;
+import org.apache.activemq.artemis.core.paging.cursor.impl.PageCursorProviderImpl;
+import org.apache.activemq.artemis.core.paging.cursor.impl.PagePositionImpl;
+import org.apache.activemq.artemis.core.paging.impl.PagingStoreFactoryNIO;
+import org.apache.activemq.artemis.core.persistence.OperationContext;
+import org.apache.activemq.artemis.core.persistence.StorageManager;
+import org.apache.activemq.artemis.core.persistence.impl.journal.AckDescribe;
+import org.apache.activemq.artemis.core.persistence.impl.journal.DescribeJournal;
+import org.apache.activemq.artemis.core.persistence.impl.journal.DescribeJournal.ReferenceDescribe;
+import org.apache.activemq.artemis.core.persistence.impl.journal.JournalRecordIds;
+import org.apache.activemq.artemis.core.persistence.impl.journal.OperationContextImpl;
+import org.apache.activemq.artemis.core.server.ActiveMQServer;
+import org.apache.activemq.artemis.core.server.Queue;
+import org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl;
+import org.apache.activemq.artemis.core.settings.impl.AddressFullMessagePolicy;
+import org.apache.activemq.artemis.core.settings.impl.AddressSettings;
+import org.apache.activemq.artemis.logs.AssertionLoggerHandler;
+import org.apache.activemq.artemis.spi.core.security.ActiveMQSecurityManagerImpl;
+import org.apache.activemq.artemis.tests.integration.IntegrationTestLogger;
+import org.apache.activemq.artemis.tests.util.ActiveMQTestBase;
+import org.jboss.logging.Logger;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class PagingTest extends ActiveMQTestBase {
+
+   private static final Logger logger = Logger.getLogger(PagingTest.class);
+
+   protected ServerLocator locator;
+   protected ActiveMQServer server;
+   protected ClientSessionFactory sf;
+   static final int MESSAGE_SIZE = 1024; // 1k
+
+   protected static final IntegrationTestLogger log = IntegrationTestLogger.LOGGER;
+
+   protected static final int RECEIVE_TIMEOUT = 5000;
+
+   protected static final int PAGE_MAX = 100 * 1024;
+
+   protected static final int PAGE_SIZE = 10 * 1024;
+
+   static final SimpleString ADDRESS = new SimpleString("SimpleAddress");
+
+   @Override
+   @Before
+   public void setUp() throws Exception {
+      super.setUp();
+      locator = createInVMNonHALocator();
+   }
+
+   @Test
+   public void testPageOnLargeMessageMultipleQueues() throws Exception {
+      Configuration config = createDefaultInVMConfig();
+
+      final int PAGE_MAX = 20 * 1024;
+
+      final int PAGE_SIZE = 10 * 1024;
+
+      ActiveMQServer server = createServer(true, config, PAGE_SIZE, PAGE_MAX);
+      server.start();
+
+      final int numberOfBytes = 1024;
+
+      locator.setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+      ClientSessionFactory sf = addSessionFactory(createSessionFactory(locator));
+
+      ClientSession session = sf.createSession(null, null, false, true, true, false, 0);
+
+      session.createQueue(ADDRESS, ADDRESS.concat("-0"), null, true);
+      session.createQueue(ADDRESS, ADDRESS.concat("-1"), null, true);
+
+      ClientProducer producer = session.createProducer(ADDRESS);
+
+      ClientMessage message = null;
+
+      for (int i = 0; i < 201; i++) {
+         message = session.createMessage(true);
+
+         message.getBodyBuffer().writerIndex(0);
+
+         message.getBodyBuffer().writeBytes(new byte[numberOfBytes]);
+
+         for (int j = 1; j <= numberOfBytes; j++) {
+            message.getBodyBuffer().writeInt(j);
+         }
+
+         producer.send(message);
+      }
+
+      session.close();
+
+      server.stop();
+
+      server = createServer(true, config, PAGE_SIZE, PAGE_MAX);
+      server.start();
+
+      sf = createSessionFactory(locator);
+
+      for (int ad = 0; ad < 2; ad++) {
+         session = sf.createSession(false, false, false);
+
+         ClientConsumer consumer = session.createConsumer(ADDRESS.concat("-" + ad));
+
+         session.start();
+
+         for (int i = 0; i < 201; i++) {
+            ClientMessage message2 = consumer.receive(10000);
+
+            Assert.assertNotNull(message2);
+
+            message2.acknowledge();
+
+            Assert.assertNotNull(message2);
+         }
+
+         if (ad > -1) {
+            session.commit();
+         }
+         else {
+            session.rollback();
+            for (int i = 0; i < 100; i++) {
+               ClientMessage message2 = consumer.receive(10000);
+
+               Assert.assertNotNull(message2);
+
+               message2.acknowledge();
+
+               Assert.assertNotNull(message2);
+            }
+            session.commit();
+
+         }
+
+         consumer.close();
+
+         session.close();
+      }
+   }
+
+   @Test
+   public void testPageCleanup() throws Exception {
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false);
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      final int numberOfMessages = 5000;
+
+      locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+      sf = createSessionFactory(locator);
+
+      ClientSession session = sf.createSession(false, false, false);
+
+      session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS, null, true);
+
+      ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
+
+      ClientMessage message = null;
+
+      byte[] body = new byte[MESSAGE_SIZE];
+
+      ByteBuffer bb = ByteBuffer.wrap(body);
+
+      for (int j = 1; j <= MESSAGE_SIZE; j++) {
+         bb.put(getSamplebyte(j));
+      }
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         message = session.createMessage(true);
+
+         ActiveMQBuffer bodyLocal = message.getBodyBuffer();
+
+         bodyLocal.writeBytes(body);
+
+         producer.send(message);
+         if (i % 1000 == 0) {
+            session.commit();
+         }
+      }
+      session.commit();
+      producer.close();
+      session.close();
+
+      session = sf.createSession(false, false, false);
+      producer = session.createProducer(PagingTest.ADDRESS);
+      producer.send(session.createMessage(true));
+      session.rollback();
+      producer.close();
+      session.close();
+
+      session = sf.createSession(false, false, false);
+      producer = session.createProducer(PagingTest.ADDRESS);
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         message = session.createMessage(true);
+
+         ActiveMQBuffer bodyLocal = message.getBodyBuffer();
+
+         bodyLocal.writeBytes(body);
+
+         producer.send(message);
+         if (i % 1000 == 0) {
+            session.commit();
+         }
+      }
+      session.commit();
+      producer.close();
+      session.close();
+      //System.out.println("Just sent " + numberOfMessages + " messages.");
+
+      Queue queue = server.locateQueue(PagingTest.ADDRESS);
+
+      session = sf.createSession(false, false, false);
+
+      session.start();
+
+      assertEquals(numberOfMessages * 2, getMessageCount(queue));
+
+      // The consumer has to be created after the getMessageCount(queue) assertion
+      // otherwise delivery could alter the messagecount and give us a false failure
+      ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS);
+      ClientMessage msg = null;
+
+      for (int i = 0; i < numberOfMessages * 2; i++) {
+         msg = consumer.receive(1000);
+         assertNotNull(msg);
+         msg.acknowledge();
+         if (i % 500 == 0) {
+            session.commit();
+         }
+      }
+      session.commit();
+      consumer.close();
+      session.close();
+
+      sf.close();
+
+      locator.close();
+
+      assertEquals(0, getMessageCount(queue));
+
+      waitForNotPaging(queue);
+
+      server.stop();
+
+      HashMap<Integer, AtomicInteger> counts = countJournalLivingRecords(server.getConfiguration());
+
+      AtomicInteger pgComplete = counts.get((int) JournalRecordIds.PAGE_CURSOR_COMPLETE);
+
+      assertTrue(pgComplete == null || pgComplete.get() == 0);
+
+      System.out.println("pgComplete = " + pgComplete);
+   }
+
+   // First page is complete but it wasn't deleted
+   @Test
+   public void testFirstPageCompleteNotDeleted() throws Exception {
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false);
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      final int numberOfMessages = 20;
+
+      locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+      sf = createSessionFactory(locator);
+
+      ClientSession session = sf.createSession(false, true, true);
+
+      Queue queue = server.createQueue(ADDRESS, ADDRESS, null, true, false);
+
+      queue.getPageSubscription().getPagingStore().startPaging();
+
+      ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
+
+      ClientMessage message = null;
+
+      byte[] body = new byte[MESSAGE_SIZE];
+
+      ByteBuffer bb = ByteBuffer.wrap(body);
+
+      for (int j = 1; j <= MESSAGE_SIZE; j++) {
+         bb.put(getSamplebyte(j));
+      }
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         message = session.createMessage(true);
+
+         ActiveMQBuffer bodyLocal = message.getBodyBuffer();
+
+         bodyLocal.writeBytes(body);
+
+         message.putIntProperty("count", i);
+
+         producer.send(message);
+
+         if ((i + 1) % 5 == 0) {
+            session.commit();
+            queue.getPageSubscription().getPagingStore().forceAnotherPage();
+         }
+      }
+
+      session.commit();
+      producer.close();
+      session.close();
+
+      // This will make the cursor to set the page complete and not actually delete it
+      queue.getPageSubscription().getPagingStore().disableCleanup();
+
+      session = sf.createSession(false, false, false);
+
+      ClientConsumer consumer = session.createConsumer(ADDRESS);
+      session.start();
+
+      for (int i = 0; i < 5; i++) {
+         ClientMessage msg = consumer.receive(2000);
+         assertNotNull(msg);
+         assertEquals(i, msg.getIntProperty("count").intValue());
+
+         msg.individualAcknowledge();
+
+         System.out.println(msg);
+      }
+
+      session.commit();
+
+      session.close();
+
+      server.stop();
+
+      server.start();
+
+      sf = createSessionFactory(locator);
+
+      session = sf.createSession(false, false, false);
+
+      consumer = session.createConsumer(ADDRESS);
+      session.start();
+
+      for (int i = 5; i < numberOfMessages; i++) {
+         ClientMessage msg = consumer.receive(2000);
+         assertNotNull(msg);
+         assertEquals(i, msg.getIntProperty("count").intValue());
+         msg.acknowledge();
+         System.out.println(msg);
+      }
+
+      assertNull(consumer.receiveImmediate());
+      session.commit();
+
+      session.close();
+      sf.close();
+      locator.close();
+
+   }
+
+   @Test
+   public void testPreparedACKAndRestart() throws Exception {
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false);
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      final int numberOfMessages = 50;
+
+      locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true).setAckBatchSize(0);
+
+      sf = createSessionFactory(locator);
+
+      ClientSession session = sf.createSession(false, true, true);
+
+      session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS, null, true);
+
+      Queue queue = server.locateQueue(PagingTest.ADDRESS);
+
+      ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
+
+      byte[] body = new byte[MESSAGE_SIZE];
+
+      ByteBuffer bb = ByteBuffer.wrap(body);
+
+      for (int j = 1; j <= MESSAGE_SIZE; j++) {
+         bb.put(getSamplebyte(j));
+      }
+
+      queue.getPageSubscription().getPagingStore().startPaging();
+
+      forcePage(queue);
+
+      // Send many messages, 5 on each page
+      for (int i = 0; i < numberOfMessages; i++) {
+         ClientMessage message = session.createMessage(true);
+
+         message.putIntProperty("count", i);
+
+         ActiveMQBuffer bodyLocal = message.getBodyBuffer();
+
+         bodyLocal.writeBytes(body);
+
+         producer.send(message);
+
+         if ((i + 1) % 5 == 0) {
+            System.out.println("Forcing at " + i);
+            session.commit();
+            queue.getPageSubscription().getPagingStore().forceAnotherPage();
+         }
+      }
+
+      session.close();
+
+      session = sf.createSession(true, false, false);
+
+      Xid xidConsumeNoCommit = newXID();
+      session.start(xidConsumeNoCommit, XAResource.TMNOFLAGS);
+
+      ClientConsumer cons = session.createConsumer(ADDRESS);
+
+      session.start();
+
+      // First message is consumed, prepared, will be rolled back later
+      ClientMessage firstMessageConsumed = cons.receive(5000);
+      assertNotNull(firstMessageConsumed);
+      firstMessageConsumed.acknowledge();
+
+      session.end(xidConsumeNoCommit, XAResource.TMSUCCESS);
+
+      session.prepare(xidConsumeNoCommit);
+
+      Xid xidConsumeCommit = newXID();
+      session.start(xidConsumeCommit, XAResource.TMNOFLAGS);
+
+      Xid neverCommittedXID = newXID();
+
+      for (int i = 1; i < numberOfMessages; i++) {
+         if (i == 20) {
+            // I elected a single message to be in prepared state, it won't ever be committed
+            session.end(xidConsumeCommit, XAResource.TMSUCCESS);
+            session.commit(xidConsumeCommit, true);
+            session.start(neverCommittedXID, XAResource.TMNOFLAGS);
+         }
+         ClientMessage message = cons.receive(5000);
+         assertNotNull(message);
+         System.out.println("ACK " + i);
+         message.acknowledge();
+         assertEquals(i, message.getIntProperty("count").intValue());
+         if (i == 20) {
+            session.end(neverCommittedXID, XAResource.TMSUCCESS);
+            session.prepare(neverCommittedXID);
+            xidConsumeCommit = newXID();
+            session.start(xidConsumeCommit, XAResource.TMNOFLAGS);
+         }
+      }
+
+      session.end(xidConsumeCommit, XAResource.TMSUCCESS);
+
+      session.commit(xidConsumeCommit, true);
+
+      session.close();
+      sf.close();
+
+      // Restart the server, and we expect cleanup to not destroy any page with prepared data
+      server.stop();
+
+      server.start();
+
+      sf = createSessionFactory(locator);
+
+      session = sf.createSession(false, true, true);
+
+      queue = server.locateQueue(ADDRESS);
+
+      assertTrue(queue.getPageSubscription().getPagingStore().isPaging());
+
+      producer = session.createProducer(ADDRESS);
+
+      for (int i = numberOfMessages; i < numberOfMessages * 2; i++) {
+         ClientMessage message = session.createMessage(true);
+
+         message.putIntProperty("count", i);
+
+         ActiveMQBuffer bodyLocal = message.getBodyBuffer();
+
+         bodyLocal.writeBytes(body);
+
+         producer.send(message);
+
+         if ((i + 1) % 5 == 0) {
+            session.commit();
+            queue.getPageSubscription().getPagingStore().forceAnotherPage();
+         }
+      }
+
+      cons = session.createConsumer(ADDRESS);
+
+      session.start();
+
+      for (int i = numberOfMessages; i < numberOfMessages * 2; i++) {
+         ClientMessage message = cons.receive(5000);
+         assertNotNull(message);
+         assertEquals(i, message.getIntProperty("count").intValue());
+         message.acknowledge();
+      }
+      assertNull(cons.receiveImmediate());
+      session.commit();
+
+      System.out.println("count = " + getMessageCount(queue));
+
+      session.commit();
+
+      session.close();
+
+      session = sf.createSession(true, false, false);
+
+      session.rollback(xidConsumeNoCommit);
+
+      session.start();
+
+      xidConsumeCommit = newXID();
+
+      session.start(xidConsumeCommit, XAResource.TMNOFLAGS);
+      cons = session.createConsumer(ADDRESS);
+
+      session.start();
+
+      ClientMessage message = cons.receive(5000);
+      assertNotNull(message);
+      message.acknowledge();
+
+      session.end(xidConsumeCommit, XAResource.TMSUCCESS);
+
+      session.commit(xidConsumeCommit, true);
+
+      session.close();
+   }
+
+   /**
+    * @param queue
+    * @throws InterruptedException
+    */
+   private void forcePage(Queue queue) throws InterruptedException {
+      for (long timeout = System.currentTimeMillis() + 5000; timeout > System.currentTimeMillis() && !queue.getPageSubscription().getPagingStore().isPaging(); ) {
+         Thread.sleep(10);
+      }
+      assertTrue(queue.getPageSubscription().getPagingStore().isPaging());
+   }
+
+   @Test
+   public void testMoveExpire() throws Exception {
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig().setJournalDirectory(getJournalDir()).setJournalSyncNonTransactional(false).setJournalCompactMinFiles(0) // disable compact
+         .setMessageExpiryScanPeriod(500);
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      AddressSettings defaultSetting = new AddressSettings().setPageSizeBytes(PAGE_SIZE).setMaxSizeBytes(PAGE_MAX).setExpiryAddress(new SimpleString("EXP")).setAddressFullMessagePolicy(AddressFullMessagePolicy.PAGE);
+
+      server.getAddressSettingsRepository().clear();
+
+      server.getAddressSettingsRepository().addMatch("#", defaultSetting);
+
+      server.start();
+
+      final int numberOfMessages = 5000;
+
+      locator = createInVMNonHALocator().setConsumerWindowSize(10 * 1024 * 1024).setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+      ClientSessionFactory sf = locator.createSessionFactory();
+
+      ClientSession session = sf.createSession(false, false, false);
+
+      session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS, null, true);
+
+      session.createQueue("EXP", "EXP", null, true);
+
+      Queue queue1 = server.locateQueue(ADDRESS);
+      Queue qEXP = server.locateQueue(new SimpleString("EXP"));
+
+      ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
+
+      final int MESSAGE_SIZE = 1024;
+
+      byte[] body = new byte[MESSAGE_SIZE];
+
+      ByteBuffer bb = ByteBuffer.wrap(body);
+
+      for (int j = 1; j <= MESSAGE_SIZE; j++) {
+         bb.put(getSamplebyte(j));
+      }
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         ClientMessage message = session.createMessage(true);
+
+         if (i < 1000) {
+            message.setExpiration(System.currentTimeMillis() + 1000);
+         }
+
+         message.putIntProperty("tst-count", i);
+
+         ActiveMQBuffer bodyLocal = message.getBodyBuffer();
+
+         bodyLocal.writeBytes(body);
+
+         producer.send(message);
+         if (i % 1000 == 0) {
+            session.commit();
+         }
+      }
+      session.commit();
+      producer.close();
+
+      for (long timeout = System.currentTimeMillis() + 60000; timeout > System.currentTimeMillis() && getMessageCount(qEXP) < 1000; ) {
+         System.out.println("count = " + getMessageCount(qEXP));
+         Thread.sleep(100);
+      }
+
+      assertEquals(1000, getMessageCount(qEXP));
+
+      session.start();
+
+      ClientConsumer consumer = session.createConsumer(ADDRESS);
+
+      for (int i = 0; i < numberOfMessages - 1000; i++) {
+         ClientMessage message = consumer.receive(5000);
+         assertNotNull(message);
+         message.acknowledge();
+         assertTrue(message.getIntProperty("tst-count") >= 1000);
+      }
+
+      session.commit();
+
+      assertNull(consumer.receiveImmediate());
+
+      for (long timeout = System.currentTimeMillis() + 5000; timeout > System.currentTimeMillis() && getMessageCount(queue1) != 0; ) {
+         Thread.sleep(100);
+      }
+      assertEquals(0, getMessageCount(queue1));
+
+      consumer.close();
+
+      consumer = session.createConsumer("EXP");
+
+      for (int i = 0; i < 1000; i++) {
+         ClientMessage message = consumer.receive(5000);
+         assertNotNull(message);
+         message.acknowledge();
+         assertTrue(message.getIntProperty("tst-count") < 1000);
+      }
+
+      assertNull(consumer.receiveImmediate());
+
+      // This is just to hold some messages as being delivered
+      ClientConsumerInternal cons = (ClientConsumerInternal) session.createConsumer(ADDRESS);
+
+      session.commit();
+      producer.close();
+      session.close();
+
+      server.stop();
+   }
+
+   @Test
+   public void testDeleteQueueRestart() throws Exception {
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig().setJournalDirectory(getJournalDir()).setJournalSyncNonTransactional(false).setJournalCompactMinFiles(0); // disable compact
+
+      ActiveMQServer server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      final int numberOfMessages = 5000;
+
+      locator = createInVMNonHALocator().setConsumerWindowSize(10 * 1024 * 1024).setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+      SimpleString QUEUE2 = ADDRESS.concat("-2");
+
+      ClientSessionFactory sf = locator.createSessionFactory();
+
+      ClientSession session = sf.createSession(false, false, false);
+
+      session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS, null, true);
+
+      session.createQueue(PagingTest.ADDRESS, QUEUE2, null, true);
+
+      ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
+
+      // This is just to hold some messages as being delivered
+      ClientConsumerInternal cons = (ClientConsumerInternal) session.createConsumer(ADDRESS);
+      ClientConsumerInternal cons2 = (ClientConsumerInternal) session.createConsumer(QUEUE2);
+
+      ClientMessage message = null;
+
+      byte[] body = new byte[MESSAGE_SIZE];
+
+      ByteBuffer bb = ByteBuffer.wrap(body);
+
+      for (int j = 1; j <= MESSAGE_SIZE; j++) {
+         bb.put(getSamplebyte(j));
+      }
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         message = session.createMessage(true);
+
+         ActiveMQBuffer bodyLocal = message.getBodyBuffer();
+
+         bodyLocal.writeBytes(body);
+
+         producer.send(message);
+         if (i % 1000 == 0) {
+            session.commit();
+         }
+      }
+      session.commit();
+      producer.close();
+      session.start();
+
+      long timeout = System.currentTimeMillis() + 30000;
+
+      // I want the buffer full to make sure there are pending messages on the server's side
+      while (System.currentTimeMillis() < timeout && (cons.getBufferSize() < 1000 || cons2.getBufferSize() < 1000)) {
+         System.out.println("cons1 buffer = " + cons.getBufferSize() + ", cons2 buffer = " + cons2.getBufferSize());
+         Thread.sleep(100);
+      }
+
+      assertTrue(cons.getBufferSize() >= 1000);
+      assertTrue(cons2.getBufferSize() >= 1000);
+
+      session.close();
+
+      Queue queue = server.locateQueue(QUEUE2);
+
+      long deletedQueueID = queue.getID();
+
+      server.destroyQueue(QUEUE2);
+
+      sf.close();
+      locator.close();
+      locator = null;
+      sf = null;
+
+      server.stop();
+
+      final HashMap<Integer, AtomicInteger> recordsType = countJournal(config);
+
+      for (Map.Entry<Integer, AtomicInteger> entry : recordsType.entrySet()) {
+         System.out.println(entry.getKey() + "=" + entry.getValue());
+      }
+
+      assertNull("The system is acking page records instead of just delete data", recordsType.get(new Integer(JournalRecordIds.ACKNOWLEDGE_CURSOR)));
+
+      Pair<List<RecordInfo>, List<PreparedTransactionInfo>> journalData = loadMessageJournal(config);
+
+      HashSet<Long> deletedQueueReferences = new HashSet<>();
+
+      for (RecordInfo info : journalData.getA()) {
+         if (info.getUserRecordType() == JournalRecordIds.ADD_REF) {
+            DescribeJournal.ReferenceDescribe ref = (ReferenceDescribe) DescribeJournal.newObjectEncoding(info);
+
+            if (ref.refEncoding.queueID == deletedQueueID) {
+               deletedQueueReferences.add(new Long(info.id));
+            }
+         }
+         else if (info.getUserRecordType() == JournalRecordIds.ACKNOWLEDGE_REF) {
+            AckDescribe ref = (AckDescribe) DescribeJournal.newObjectEncoding(info);
+
+            if (ref.refEncoding.queueID == deletedQueueID) {
+               deletedQueueReferences.remove(new Long(info.id));
+            }
+         }
+      }
+
+      if (!deletedQueueReferences.isEmpty()) {
+         for (Long value : deletedQueueReferences) {
+            System.out.println("Deleted Queue still has a reference:" + value);
+         }
+
+         fail("Deleted queue still have references");
+      }
+
+      server.start();
+
+      locator = createInVMNonHALocator();
+      locator.setConsumerWindowSize(10 * 1024 * 1024);
+      sf = locator.createSessionFactory();
+      session = sf.createSession(false, false, false);
+      cons = (ClientConsumerInternal) session.createConsumer(ADDRESS);
+      session.start();
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         message = cons.receive(5000);
+         assertNotNull(message);
+         message.acknowledge();
+         if (i % 1000 == 0) {
+            session.commit();
+         }
+      }
+      session.commit();
+      producer.close();
+      session.close();
+
+      queue = server.locateQueue(PagingTest.ADDRESS);
+
+      assertEquals(0, getMessageCount(queue));
+
+      timeout = System.currentTimeMillis() + 10000;
+      while (timeout > System.currentTimeMillis() && queue.getPageSubscription().getPagingStore().isPaging()) {
+         Thread.sleep(100);
+      }
+      assertFalse(queue.getPageSubscription().getPagingStore().isPaging());
+
+      server.stop();
+   }
+
+   @Test
+   public void testPreparePersistent() throws Exception {
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false);
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      final int numberOfMessages = 5000;
+
+      final int numberOfTX = 10;
+
+      final int messagesPerTX = numberOfMessages / numberOfTX;
+
+      locator = createInVMNonHALocator();
+
+      locator.setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+      sf = createSessionFactory(locator);
+
+      ClientSession session = sf.createSession(false, false, false);
+
+      session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS, null, true);
+
+      ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
+
+      ClientMessage message = null;
+
+      byte[] body = new byte[MESSAGE_SIZE];
+
+      ByteBuffer bb = ByteBuffer.wrap(body);
+
+      for (int j = 1; j <= MESSAGE_SIZE; j++) {
+         bb.put(getSamplebyte(j));
+      }
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         message = session.createMessage(true);
+
+         ActiveMQBuffer bodyLocal = message.getBodyBuffer();
+
+         bodyLocal.writeBytes(body);
+
+         message.putIntProperty(new SimpleString("id"), i);
+
+         producer.send(message);
+         if (i % 1000 == 0) {
+            session.commit();
+         }
+      }
+      session.commit();
+      session.close();
+      session = null;
+
+      sf.close();
+      locator.close();
+
+      server.stop();
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+      server.start();
+
+      locator = createInVMNonHALocator();
+      sf = createSessionFactory(locator);
+
+      Queue queue = server.locateQueue(ADDRESS);
+
+      assertEquals(numberOfMessages, getMessageCount(queue));
+
+      LinkedList<Xid> xids = new LinkedList<>();
+
+      int msgReceived = 0;
+      for (int i = 0; i < numberOfTX; i++) {
+         ClientSession sessionConsumer = sf.createSession(true, false, false);
+         Xid xid = newXID();
+         xids.add(xid);
+         sessionConsumer.start(xid, XAResource.TMNOFLAGS);
+         sessionConsumer.start();
+         ClientConsumer consumer = sessionConsumer.createConsumer(PagingTest.ADDRESS);
+         for (int msgCount = 0; msgCount < messagesPerTX; msgCount++) {
+            if (msgReceived == numberOfMessages) {
+               break;
+            }
+            msgReceived++;
+            ClientMessage msg = consumer.receive(10000);
+            assertNotNull(msg);
+            msg.acknowledge();
+         }
+         sessionConsumer.end(xid, XAResource.TMSUCCESS);
+         sessionConsumer.prepare(xid);
+         sessionConsumer.close();
+      }
+
+      ClientSession sessionCheck = sf.createSession(true, true);
+
+      ClientConsumer consumer = sessionCheck.createConsumer(PagingTest.ADDRESS);
+
+      assertNull(consumer.receiveImmediate());
+
+      sessionCheck.close();
+
+      assertEquals(numberOfMessages, getMessageCount(queue));
+
+      sf.close();
+      locator.close();
+
+      server.stop();
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+      server.start();
+
+      waitForServerToStart(server);
+
+      queue = server.locateQueue(ADDRESS);
+
+      locator = createInVMNonHALocator();
+      sf = createSessionFactory(locator);
+
+      session = sf.createSession(true, false, false);
+
+      consumer = session.createConsumer(PagingTest.ADDRESS);
+
+      session.start();
+
+      assertEquals(numberOfMessages, getMessageCount(queue));
+
+      ClientMessage msg = consumer.receive(5000);
+      if (msg != null) {
+         while (true) {
+            ClientMessage msg2 = consumer.receive(1000);
+            if (msg2 == null) {
+               break;
+            }
+         }
+      }
+      assertNull(msg);
+
+      for (int i = xids.size() - 1; i >= 0; i--) {
+         Xid xid = xids.get(i);
+         session.rollback(xid);
+      }
+
+      xids.clear();
+
+      session.close();
+
+      session = sf.createSession(false, false, false);
+
+      session.start();
+
+      consumer = session.createConsumer(PagingTest.ADDRESS);
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         msg = consumer.receive(1000);
+         assertNotNull(msg);
+         msg.acknowledge();
+
+         assertEquals(i, msg.getIntProperty("id").intValue());
+
+         if (i % 500 == 0) {
+            session.commit();
+         }
+      }
+
+      session.commit();
+
+      session.close();
+
+      sf.close();
+
+      locator.close();
+
+      assertEquals(0, getMessageCount(queue));
+
+      waitForNotPaging(queue);
+   }
+
+   @Test
+   public void testSendOverBlockingNoFlowControl() throws Exception {
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false);
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+      server.getAddressSettingsRepository().getMatch("#").setAddressFullMessagePolicy(AddressFullMessagePolicy.BLOCK);
+
+      server.start();
+
+      final int biggerMessageSize = 10 * 1024;
+
+      final int numberOfMessages = 500;
+
+      locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true).setProducerWindowSize(-1).setMinLargeMessageSize(1024 * 1024);
+
+      sf = createSessionFactory(locator);
+
+      ClientSession session = sf.createSession(false, false, false);
+
+      session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS, null, true);
+
+      ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
+
+      ClientMessage message = null;
+
+      byte[] body = new byte[biggerMessageSize];
+
+      ByteBuffer bb = ByteBuffer.wrap(body);
+
+      for (int j = 1; j <= biggerMessageSize; j++) {
+         bb.put(getSamplebyte(j));
+      }
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         message = session.createMessage(true);
+
+         ActiveMQBuffer bodyLocal = message.getBodyBuffer();
+
+         bodyLocal.writeBytes(body);
+
+         message.putIntProperty(new SimpleString("id"), i);
+
+         producer.send(message);
+
+         if (i % 10 == 0) {
+            session.commit();
+         }
+      }
+      session.commit();
+
+      session.start();
+
+      ClientConsumer cons = session.createConsumer(ADDRESS);
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         message = cons.receive(5000);
+         assertNotNull(message);
+         message.acknowledge();
+
+         if (i % 10 == 0) {
+            session.commit();
+         }
+      }
+
+      session.commit();
+
+   }
+
+   @Test
+   public void testReceiveImmediate() throws Exception {
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false);
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      final int numberOfMessages = 1000;
+
+      locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+      sf = createSessionFactory(locator);
+
+      ClientSession session = sf.createSession(false, false, false);
+
+      session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS, null, true);
+
+      ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
+
+      ClientMessage message = null;
+
+      byte[] body = new byte[MESSAGE_SIZE];
+
+      ByteBuffer bb = ByteBuffer.wrap(body);
+
+      for (int j = 1; j <= MESSAGE_SIZE; j++) {
+         bb.put(getSamplebyte(j));
+      }
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         message = session.createMessage(true);
+
+         ActiveMQBuffer bodyLocal = message.getBodyBuffer();
+
+         bodyLocal.writeBytes(body);
+
+         message.putIntProperty(new SimpleString("id"), i);
+
+         producer.send(message);
+         if (i % 1000 == 0) {
+            session.commit();
+         }
+      }
+      session.commit();
+      session.close();
+
+      session = null;
+
+      sf.close();
+      locator.close();
+
+      server.stop();
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+      server.start();
+
+      locator = createInVMNonHALocator();
+      sf = createSessionFactory(locator);
+
+      Queue queue = server.locateQueue(ADDRESS);
+
+      assertEquals(numberOfMessages, getMessageCount(queue));
+
+      int msgReceived = 0;
+      ClientSession sessionConsumer = sf.createSession(false, false, false);
+      sessionConsumer.start();
+      ClientConsumer consumer = sessionConsumer.createConsumer(PagingTest.ADDRESS);
+      for (int msgCount = 0; msgCount < numberOfMessages; msgCount++) {
+         log.info("Received " + msgCount);
+         msgReceived++;
+         ClientMessage msg = consumer.receiveImmediate();
+         if (msg == null) {
+            log.info("It's null. leaving now");
+            sessionConsumer.commit();
+            fail("Didn't receive a message");
+         }
+         msg.acknowledge();
+
+         if (msgCount % 5 == 0) {
+            log.info("commit");
+            sessionConsumer.commit();
+         }
+      }
+
+      sessionConsumer.commit();
+
+      sessionConsumer.close();
+
+      sf.close();
+
+      locator.close();
+
+      assertEquals(0, getMessageCount(queue));
+
+      long timeout = System.currentTimeMillis() + 5000;
+      while (timeout > System.currentTimeMillis() && queue.getPageSubscription().getPagingStore().isPaging()) {
+         Thread.sleep(100);
+      }
+      assertFalse(queue.getPageSubscription().getPagingStore().isPaging());
+
+   }
+
+   /**
+    * This test will remove all the page directories during a restart, simulating a crash scenario. The server should still start after this
+    */
+   @Test
+   public void testDeletePhysicalPages() throws Exception {
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig().setPersistDeliveryCountBeforeDelivery(true);
+
+      config.setJournalSyncNonTransactional(false);
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      final int numberOfMessages = 1000;
+
+      locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+      sf = createSessionFactory(locator);
+
+      ClientSession session = sf.createSession(false, false, false);
+
+      session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS, null, true);
+
+      ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
+
+      ClientMessage message = null;
+
+      byte[] body = new byte[MESSAGE_SIZE];
+
+      ByteBuffer bb = ByteBuffer.wrap(body);
+
+      for (int j = 1; j <= MESSAGE_SIZE; j++) {
+         bb.put(getSamplebyte(j));
+      }
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         message = session.createMessage(true);
+
+         ActiveMQBuffer bodyLocal = message.getBodyBuffer();
+
+         bodyLocal.writeBytes(body);
+
+         message.putIntProperty(new SimpleString("id"), i);
+
+         producer.send(message);
+         if (i % 1000 == 0) {
+            session.commit();
+         }
+      }
+      session.commit();
+      session.close();
+
+      session = null;
+
+      sf.close();
+      locator.close();
+
+      server.stop();
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+      server.start();
+
+      locator = createInVMNonHALocator();
+      sf = createSessionFactory(locator);
+
+      Queue queue = server.locateQueue(ADDRESS);
+
+      assertEquals(numberOfMessages, getMessageCount(queue));
+
+      int msgReceived = 0;
+      ClientSession sessionConsumer = sf.createSession(false, false, false);
+      sessionConsumer.start();
+      ClientConsumer consumer = sessionConsumer.createConsumer(PagingTest.ADDRESS);
+      for (int msgCount = 0; msgCount < numberOfMessages; msgCount++) {
+         log.info("Received " + msgCount);
+         msgReceived++;
+         ClientMessage msg = consumer.receiveImmediate();
+         if (msg == null) {
+            log.info("It's null. leaving now");
+            sessionConsumer.commit();
+            fail("Didn't receive a message");
+         }
+         msg.acknowledge();
+
+         if (msgCount % 5 == 0) {
+            log.info("commit");
+            sessionConsumer.commit();
+         }
+      }
+
+      sessionConsumer.commit();
+
+      sessionConsumer.close();
+
+      sf.close();
+
+      locator.close();
+
+      assertEquals(0, getMessageCount(queue));
+
+      long timeout = System.currentTimeMillis() + 5000;
+      while (timeout > System.currentTimeMillis() && queue.getPageSubscription().getPagingStore().isPaging()) {
+         Thread.sleep(100);
+      }
+      assertFalse(queue.getPageSubscription().getPagingStore().isPaging());
+
+      server.stop();
+
+      // Deleting the paging data. Simulating a failure
+      // a dumb user, or anything that will remove the data
+      deleteDirectory(new File(getPageDir()));
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+      server.start();
+
+      locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+      sf = createSessionFactory(locator);
+
+      queue = server.locateQueue(ADDRESS);
+
+      sf = createSessionFactory(locator);
+      session = sf.createSession(false, false, false);
+
+      producer = session.createProducer(PagingTest.ADDRESS);
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         message = session.createMessage(true);
+
+         ActiveMQBuffer bodyLocal = message.getBodyBuffer();
+
+         bodyLocal.writeBytes(body);
+
+         message.putIntProperty(new SimpleString("id"), i);
+
+         producer.send(message);
+         if (i % 1000 == 0) {
+            session.commit();
+         }
+      }
+
+      session.commit();
+
+      server.stop();
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+      server.start();
+
+      locator = createInVMNonHALocator();
+      sf = createSessionFactory(locator);
+
+      queue = server.locateQueue(ADDRESS);
+
+      // assertEquals(numberOfMessages, getMessageCount(queue));
+
+      msgReceived = 0;
+      sessionConsumer = sf.createSession(false, false, false);
+      sessionConsumer.start();
+      consumer = sessionConsumer.createConsumer(PagingTest.ADDRESS);
+      for (int msgCount = 0; msgCount < numberOfMessages; msgCount++) {
+         log.info("Received " + msgCount);
+         msgReceived++;
+         ClientMessage msg = consumer.receiveImmediate();
+         if (msg == null) {
+            log.info("It's null. leaving now");
+            sessionConsumer.commit();
+            fail("Didn't receive a message");
+         }
+         msg.acknowledge();
+
+         if (msgCount % 5 == 0) {
+            log.info("commit");
+            sessionConsumer.commit();
+         }
+      }
+
+      sessionConsumer.commit();
+
+      sessionConsumer.close();
+
+   }
+
+   @Test
+   public void testMissingTXEverythingAcked() throws Exception {
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false);
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      final int numberOfMessages = 5000;
+
+      final int numberOfTX = 10;
+
+      final int messagesPerTX = numberOfMessages / numberOfTX;
+
+      try {
+         locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+         sf = createSessionFactory(locator);
+
+         ClientSession session = sf.createSession(false, false, false);
+
+         session.createQueue(ADDRESS.toString(), "q1", true);
+
+         session.createQueue(ADDRESS.toString(), "q2", true);
+
+         ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
+
+         ClientMessage message = null;
+
+         byte[] body = new byte[MESSAGE_SIZE];
+
+         ByteBuffer bb = ByteBuffer.wrap(body);
+
+         for (int j = 1; j <= MESSAGE_SIZE; j++) {
+            bb.put(getSamplebyte(j));
+         }
+
+         for (int i = 0; i < numberOfMessages; i++) {
+            message = session.createMessage(true);
+
+            ActiveMQBuffer bodyLocal = message.getBodyBuffer();
+
+            bodyLocal.writeBytes(body);
+
+            message.putIntProperty(new SimpleString("id"), i);
+
+            producer.send(message);
+            if (i % messagesPerTX == 0) {
+               session.commit();
+            }
+         }
+         session.commit();
+         session.close();
+      }
+      finally {
+         try {
+            server.stop();
+         }
+         catch (Throwable ignored) {
+         }
+      }
+
+      ArrayList<RecordInfo> records = new ArrayList<>();
+
+      List<PreparedTransactionInfo> list = new ArrayList<>();
+
+      JournalImpl jrn = new JournalImpl(config.getJournalFileSize(), 2, 2, 0, 0, new NIOSequentialFileFactory(server.getConfiguration().getJournalLocation(), 1), "activemq-data", "amq", 1);
+      jrn.start();
+      jrn.load(records, list, null);
+
+      // Delete everything from the journal
+      for (RecordInfo info : records) {
+         if (!info.isUpdate && info.getUserRecordType() != JournalRecordIds.PAGE_CURSOR_COUNTER_VALUE &&
+            info.getUserRecordType() != JournalRecordIds.PAGE_CURSOR_COUNTER_INC &&
+            info.getUserRecordType() != JournalRecordIds.PAGE_CURSOR_COMPLETE) {
+            jrn.appendDeleteRecord(info.id, false);
+         }
+      }
+
+      jrn.stop();
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      ClientSessionFactory csf = createSessionFactory(locator);
+
+      ClientSession sess = csf.createSession();
+
+      sess.start();
+
+      ClientConsumer cons = sess.createConsumer("q1");
+
+      assertNull(cons.receiveImmediate());
+
+      ClientConsumer cons2 = sess.createConsumer("q2");
+      assertNull(cons2.receiveImmediate());
+
+      Queue q1 = server.locateQueue(new SimpleString("q1"));
+      Queue q2 = server.locateQueue(new SimpleString("q2"));
+
+      q1.getPageSubscription().cleanupEntries(false);
+      q2.getPageSubscription().cleanupEntries(false);
+
+      PageCursorProvider provider = q1.getPageSubscription().getPagingStore().getCursorProvider();
+      provider.cleanup();
+
+      waitForNotPaging(q1);
+
+      sess.close();
+   }
+
+   @Test
+   public void testMissingTXEverythingAcked2() throws Exception {
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false);
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      final int numberOfMessages = 6;
+
+      final int numberOfTX = 2;
+
+      final int messagesPerTX = numberOfMessages / numberOfTX;
+
+      try {
+         locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+         sf = createSessionFactory(locator);
+
+         ClientSession session = sf.createSession(false, false, false);
+
+         session.createQueue(ADDRESS.toString(), "q1", true);
+
+         session.createQueue(ADDRESS.toString(), "q2", true);
+
+         server.getPagingManager().getPageStore(ADDRESS).startPaging();
+
+         ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
+
+         ClientMessage message = null;
+
+         byte[] body = new byte[MESSAGE_SIZE];
+
+         ByteBuffer bb = ByteBuffer.wrap(body);
+
+         for (int j = 1; j <= MESSAGE_SIZE; j++) {
+            bb.put(getSamplebyte(j));
+         }
+
+         for (int i = 0; i < numberOfMessages; i++) {
+            message = session.createMessage(true);
+
+            ActiveMQBuffer bodyLocal = message.getBodyBuffer();
+
+            bodyLocal.writeBytes(body);
+
+            message.putStringProperty("id", "str-" + i);
+
+            producer.send(message);
+            if ((i + 1) % messagesPerTX == 0) {
+               session.commit();
+            }
+         }
+         session.commit();
+
+         session.start();
+
+         for (int i = 1; i <= 2; i++) {
+            ClientConsumer cons = session.createConsumer("q" + i);
+
+            for (int j = 0; j < 3; j++) {
+               ClientMessage msg = cons.receive(5000);
+
+               assertNotNull(msg);
+
+               assertEquals("str-" + j, msg.getStringProperty("id"));
+
+               msg.acknowledge();
+            }
+
+            session.commit();
+
+         }
+
+         session.close();
+      }
+      finally {
+         locator.close();
+         try {
+            server.stop();
+         }
+         catch (Throwable ignored) {
+         }
+      }
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      locator = createInVMNonHALocator();
+
+      locator.setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+      ClientSessionFactory csf = createSessionFactory(locator);
+
+      ClientSession session = csf.createSession();
+
+      session.start();
+
+      for (int i = 1; i <= 2; i++) {
+         ClientConsumer cons = session.createConsumer("q" + i);
+
+         for (int j = 3; j < 6; j++) {
+            ClientMessage msg = cons.receive(5000);
+
+            assertNotNull(msg);
+
+            assertEquals("str-" + j, msg.getStringProperty("id"));
+
+            msg.acknowledge();
+         }
+
+         session.commit();
+         assertNull(cons.receive(500));
+
+      }
+
+      session.close();
+
+      long timeout = System.currentTimeMillis() + 5000;
+
+      while (System.currentTimeMillis() < timeout && server.getPagingManager().getPageStore(ADDRESS).isPaging()) {
+         Thread.sleep(100);
+      }
+   }
+
+   @Test
+   public void testTwoQueuesOneNoRouting() throws Exception {
+      boolean persistentMessages = true;
+
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false);
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      final int numberOfMessages = 1000;
+
+      locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+      sf = createSessionFactory(locator);
+
+      ClientSession session = sf.createSession(false, false, false);
+
+      session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS, null, true);
+      session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS.concat("-invalid"), new SimpleString(ActiveMQServerImpl.GENERIC_IGNORED_FILTER), true);
+
+      ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
+
+      ClientMessage message = null;
+
+      byte[] body = new byte[MESSAGE_SIZE];
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         message = session.createMessage(persistentMessages);
+
+         ActiveMQBuffer bodyLocal = message.getBodyBuffer();
+
+         bodyLocal.writeBytes(body);
+
+         message.putIntProperty(new SimpleString("id"), i);
+
+         producer.send(message);
+         if (i % 1000 == 0) {
+            session.commit();
+         }
+      }
+
+      session.commit();
+
+      session.start();
+
+      ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS);
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         message = consumer.receive(5000);
+         assertNotNull(message);
+         message.acknowledge();
+
+         assertEquals(i, message.getIntProperty("id").intValue());
+         if (i % 1000 == 0) {
+            session.commit();
+         }
+      }
+
+      session.commit();
+
+      session.commit();
+
+      session.commit();
+
+      PagingStore store = server.getPagingManager().getPageStore(ADDRESS);
+      store.getCursorProvider().cleanup();
+
+      long timeout = System.currentTimeMillis() + 5000;
+      while (store.isPaging() && timeout > System.currentTimeMillis()) {
+         Thread.sleep(100);
+      }
+
+      // It's async, so need to wait a bit for it happening
+      assertFalse(server.getPagingManager().getPageStore(ADDRESS).isPaging());
+   }
+
+   @Test
+   public void testSendReceivePagingPersistent() throws Exception {
+      internaltestSendReceivePaging(true);
+   }
+
+   @Test
+   public void testSendReceivePagingNonPersistent() throws Exception {
+      internaltestSendReceivePaging(false);
+   }
+
+   @Test
+   public void testWithDiverts() throws Exception {
+      internalMultiQueuesTest(true);
+   }
+
+   @Test
+   public void testWithMultiQueues() throws Exception {
+      internalMultiQueuesTest(false);
+   }
+
+   public void internalMultiQueuesTest(final boolean divert) throws Exception {
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false);
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      if (divert) {
+         DivertConfiguration divert1 = new DivertConfiguration().setName("dv1").setRoutingName("nm1").setAddress(PagingTest.ADDRESS.toString()).setForwardingAddress(PagingTest.ADDRESS.toString() + "-1").setExclusive(true);
+
+         config.addDivertConfiguration(divert1);
+
+         DivertConfiguration divert2 = new DivertConfiguration().setName("dv2").setRoutingName("nm2").setAddress(PagingTest.ADDRESS.toString()).setForwardingAddress(PagingTest.ADDRESS.toString() + "-2").setExclusive(true);
+
+         config.addDivertConfiguration(divert2);
+      }
+
+      server.start();
+
+      final int numberOfMessages = 3000;
+
+      final byte[] body = new byte[MESSAGE_SIZE];
+
+      ByteBuffer bb = ByteBuffer.wrap(body);
+
+      for (int j = 1; j <= MESSAGE_SIZE; j++) {
+         bb.put(getSamplebyte(j));
+      }
+
+      final AtomicBoolean running = new AtomicBoolean(true);
+
+      class TCount extends Thread {
+
+         Queue queue;
+
+         TCount(Queue queue) {
+            this.queue = queue;
+         }
+
+         @Override
+         public void run() {
+            try {
+               while (running.get()) {
+                  // this will be overusing what some users do. flush / getCount
+                  getMessagesAdded(queue);
+                  getMessageCount(queue);
+                  Thread.sleep(10);
+               }
+            }
+            catch (InterruptedException e) {
+               log.info("Thread interrupted");
+            }
+         }
+      }
+
+      TCount tcount1 = null;
+      TCount tcount2 = null;
+
+      try {
+         {
+            locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+            sf = createSessionFactory(locator);
+
+            ClientSession session = sf.createSession(false, false, false);
+
+            if (divert) {
+               session.createQueue(PagingTest.ADDRESS + "-1", PagingTest.ADDRESS + "-1", null, true);
+
+               session.createQueue(PagingTest.ADDRESS + "-2", PagingTest.ADDRESS + "-2", null, true);
+            }
+            else {
+               session.createQueue(PagingTest.ADDRESS.toString(), PagingTest.ADDRESS + "-1", null, true);
+
+               session.createQueue(PagingTest.ADDRESS.toString(), PagingTest.ADDRESS + "-2", null, true);
+            }
+
+            ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
+
+            ClientMessage message = null;
+
+            for (int i = 0; i < numberOfMessages; i++) {
+               if (i % 500 == 0) {
+                  log.info("Sent " + i + " messages");
+                  session.commit();
+               }
+               message = session.createMessage(true);
+
+               ActiveMQBuffer bodyLocal = message.getBodyBuffer();
+
+               bodyLocal.writeBytes(body);
+
+               message.putIntProperty(new SimpleString("id"), i);
+
+               producer.send(message);
+            }
+
+            session.commit();
+
+            session.close();
+
+            server.stop();
+
+            sf.close();
+            locator.close();
+         }
+
+         server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+         server.start();
+
+         Queue queue1 = server.locateQueue(PagingTest.ADDRESS.concat("-1"));
+
+         Queue queue2 = server.locateQueue(PagingTest.ADDRESS.concat("-2"));
+
+         assertNotNull(queue1);
+
+         assertNotNull(queue2);
+
+         assertNotSame(queue1, queue2);
+
+         tcount1 = new TCount(queue1);
+
+         tcount2 = new TCount(queue2);
+
+         tcount1.start();
+         tcount2.start();
+
+         locator = createInVMNonHALocator();
+         final ClientSessionFactory sf2 = createSessionFactory(locator);
+
+         final AtomicInteger errors = new AtomicInteger(0);
+
+         Thread[] threads = new Thread[2];
+
+         for (int start = 1; start <= 2; start++) {
+
+            final String addressToSubscribe = PagingTest.ADDRESS + "-" + start;
+
+            threads[start - 1] = new Thread() {
+               @Override
+               public void run() {
+                  try {
+                     ClientSession session = sf2.createSession(null, null, false, true, true, false, 0);
+
+                     ClientConsumer consumer = session.createConsumer(addressToSubscribe);
+
+                     session.start();
+
+                     for (int i = 0; i < numberOfMessages; i++) {
+                        ClientMessage message2 = consumer.receive(PagingTest.RECEIVE_TIMEOUT);
+
+                        Assert.assertNotNull(message2);
+
+                        Assert.assertEquals(i, message2.getIntProperty("id").intValue());
+
+                        message2.acknowledge();
+
+                        Assert.assertNotNull(message2);
+
+                        if (i % 100 == 0) {
+                           if (i % 5000 == 0) {
+                              log.info(addressToSubscribe + " consumed " + i + " messages");
+                           }
+                           session.commit();
+                        }
+
+                        try {
+                           assertBodiesEqual(body, message2.getBodyBuffer());
+                        }
+                        catch (AssertionError e) {
+                           PagingTest.log.info("Expected buffer:" + ActiveMQTestBase.dumpBytesHex(body, 40));
+                           PagingTest.log.info("Arriving buffer:" + ActiveMQTestBase.dumpBytesHex(message2.getBodyBuffer().toByteBuffer().array(), 40));
+                           throw e;
+                        }
+                     }
+
+                     session.commit();
+
+                     consumer.close();
+
+                     session.close();
+                  }
+                  catch (Throwable e) {
+                     e.printStackTrace();
+                     errors.incrementAndGet();
+                  }
+
+               }
+            };
+         }
+
+         for (int i = 0; i < 2; i++) {
+            threads[i].start();
+         }
+
+         for (int i = 0; i < 2; i++) {
+            threads[i].join();
+         }
+
+         sf2.close();
+         locator.close();
+
+         assertEquals(0, errors.get());
+
+         for (int i = 0; i < 20 && server.getPagingManager().getTransactions().size() != 0; i++) {
+            if (server.getPagingManager().getTransactions().size() != 0) {
+               // The delete may be asynchronous, giving some time case it eventually happen asynchronously
+               Thread.sleep(500);
+            }
+         }
+
+         assertEquals(0, server.getPagingManager().getTransactions().size());
+
+      }
+      finally {
+         running.set(false);
+
+         if (tcount1 != null) {
+            tcount1.interrupt();
+            tcount1.join();
+         }
+
+         if (tcount2 != null) {
+            tcount2.interrupt();
+            tcount2.join();
+         }
+
+         try {
+            server.stop();
+         }
+         catch (Throwable ignored) {
+         }
+      }
+
+   }
+
+   @Test
+   public void testMultiQueuesNonPersistentAndPersistent() throws Exception {
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false);
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      final int numberOfMessages = 3000;
+
+      final byte[] body = new byte[MESSAGE_SIZE];
+
+      ByteBuffer bb = ByteBuffer.wrap(body);
+
+      for (int j = 1; j <= MESSAGE_SIZE; j++) {
+         bb.put(getSamplebyte(j));
+      }
+
+      {
+         locator = createInVMNonHALocator();
+
+         locator.setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+         sf = createSessionFactory(locator);
+
+         ClientSession session = sf.createSession(false, false, false);
+
+         session.createQueue(PagingTest.ADDRESS.toString(), PagingTest.ADDRESS + "-1", null, true);
+
+         session.createQueue(PagingTest.ADDRESS.toString(), PagingTest.ADDRESS + "-2", null, false);
+
+         ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
+
+         ClientMessage message = null;
+
+         for (int i = 0; i < numberOfMessages; i++) {
+            if (i % 500 == 0) {
+               session.commit();
+            }
+            message = session.createMessage(true);
+
+            ActiveMQBuffer bodyLocal = message.getBodyBuffer();
+
+            bodyLocal.writeBytes(body);
+
+            message.putIntProperty(new SimpleString("id"), i);
+
+            producer.send(message);
+         }
+
+         session.commit();
+
+         session.close();
+
+         server.stop();
+
+         sf.close();
+         locator.close();
+      }
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+      server.start();
+
+      ServerLocator locator1 = createInVMNonHALocator();
+      final ClientSessionFactory sf2 = locator1.createSessionFactory();
+
+      final AtomicInteger errors = new AtomicInteger(0);
+
+      Thread t = new Thread() {
+         @Override
+         public void run() {
+            try {
+               ClientSession session = sf2.createSession(null, null, false, true, true, false, 0);
+
+               ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS + "-1");
+
+               session.start();
+
+               for (int i = 0; i < numberOfMessages; i++) {
+                  ClientMessage message2 = consumer.receive(PagingTest.RECEIVE_TIMEOUT);
+
+                  Assert.assertNotNull(message2);
+
+                  Assert.assertEquals(i, message2.getIntProperty("id").intValue());
+
+                  message2.acknowledge();
+
+                  Assert.assertNotNull(message2);
+
+                  if (i % 1000 == 0) {
+                     session.commit();
+                  }
+
+                  try {
+                     assertBodiesEqual(body, message2.getBodyBuffer());
+                  }
+                  catch (AssertionError e) {
+                     PagingTest.log.info("Expected buffer:" + ActiveMQTestBase.dumpBytesHex(body, 40));
+                     PagingTest.log.info("Arriving buffer:" + ActiveMQTestBase.dumpBytesHex(message2.getBodyBuffer().toByteBuffer().array(), 40));
+                     throw e;
+                  }
+               }
+
+               session.commit();
+
+               consumer.close();
+
+               session.close();
+            }
+            catch (Throwable e) {
+               e.printStackTrace();
+               errors.incrementAndGet();
+            }
+
+         }
+      };
+
+      t.start();
+      t.join();
+
+      assertEquals(0, errors.get());
+
+      for (int i = 0; i < 20 && server.getPagingManager().getPageStore(ADDRESS).isPaging(); i++) {
+         // The delete may be asynchronous, giving some time case it eventually happen asynchronously
+         Thread.sleep(500);
+      }
+
+      assertFalse(server.getPagingManager().getPageStore(ADDRESS).isPaging());
+
+      for (int i = 0; i < 20 && server.getPagingManager().getTransactions().size() != 0; i++) {
+         // The delete may be asynchronous, giving some time case it eventually happen asynchronously
+         Thread.sleep(500);
+      }
+
+      assertEquals(0, server.getPagingManager().getTransactions().size());
+
+   }
+
+   private void internaltestSendReceivePaging(final boolean persistentMessages) throws Exception {
+
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false);
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      final int numberOfIntegers = 256;
+
+      final int numberOfMessages = 1000;
+      locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+      sf = createSessionFactory(locator);
+
+      ClientSession session = sf.createSession(null, null, false, true, true, false, 0);
+
+      session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS, null, true);
+
+      Queue queue = server.locateQueue(ADDRESS);
+      queue.getPageSubscription().getPagingStore().startPaging();
+
+      ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
+
+      ClientMessage message = null;
+
+      byte[] body = new byte[numberOfIntegers * 4];
+
+      ByteBuffer bb = ByteBuffer.wrap(body);
+
+      for (int j = 1; j <= numberOfIntegers; j++) {
+         bb.putInt(j);
+      }
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         message = session.createMessage(persistentMessages);
+
+         ActiveMQBuffer bodyLocal = message.getBodyBuffer();
+
+         bodyLocal.writeBytes(body);
+
+         message.putIntProperty(new SimpleString("id"), i);
+
+         producer.send(message);
+      }
+
+      session.close();
+      sf.close();
+      locator.close();
+
+      server.stop();
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+      server.start();
+
+      locator = createInVMNonHALocator();
+      sf = createSessionFactory(locator);
+
+      session = sf.createSession(null, null, false, true, true, false, 0);
+
+      ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS);
+
+      session.start();
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         ClientMessage message2 = consumer.receive(PagingTest.RECEIVE_TIMEOUT);
+
+         Assert.assertNotNull(message2);
+
+         Assert.assertEquals(i, message2.getIntProperty("id").intValue());
+
+         assertEquals(body.length, message2.getBodySize());
+
+         message2.acknowledge();
+
+         Assert.assertNotNull(message2);
+
+         if (i % 1000 == 0) {
+            session.commit();
+         }
+
+         try {
+            assertBodiesEqual(body, message2.getBodyBuffer());
+         }
+         catch (AssertionError e) {
+            PagingTest.log.info("Expected buffer:" + ActiveMQTestBase.dumpBytesHex(body, 40));
+            PagingTest.log.info("Arriving buffer:" + ActiveMQTestBase.dumpBytesHex(message2.getBodyBuffer().toByteBuffer().array(), 40));
+            throw e;
+         }
+      }
+
+      consumer.close();
+
+      session.close();
+   }
+
+   private void assertBodiesEqual(final byte[] body, final ActiveMQBuffer buffer) {
+      byte[] other = new byte[body.length];
+
+      buffer.readBytes(other);
+
+      ActiveMQTestBase.assertEqualsByteArrays(body, other);
+   }
+
+   /**
+    * - Make a destination in page mode
+    * - Add stuff to a transaction
+    * - Consume the entire destination (not in page mode any more)
+    * - Add stuff to a transaction again
+    * - Check order
+    */
+   @Test
+   public void testDepageDuringTransaction() throws Exception {
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig();
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+      sf = createSessionFactory(locator);
+
+      ClientSession session = sf.createSession(null, null, false, true, true, false, 0);
+
+      session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS, null, true);
+
+      ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
+
+      byte[] body = new byte[MESSAGE_SIZE];
+      // ActiveMQBuffer bodyLocal = ActiveMQChannelBuffers.buffer(DataConstants.SIZE_INT * numberOfIntegers);
+
+      ClientMessage message = null;
+
+      int numberOfMessages = 0;
+      while (true) {
+         message = session.createMessage(true);
+         message.getBodyBuffer().writeBytes(body);
+
+         // Stop sending message as soon as we start paging
+         if (server.getPagingManager().getPageStore(PagingTest.ADDRESS).isPaging()) {
+            break;
+         }
+         numberOfMessages++;
+
+         producer.send(message);
+      }
+
+      Assert.assertTrue(server.getPagingManager().getPageStore(PagingTest.ADDRESS).isPaging());
+
+      session.start();
+
+      ClientSession sessionTransacted = sf.createSession(null, null, false, false, false, false, 0);
+
+      ClientProducer producerTransacted = sessionTransacted.createProducer(PagingTest.ADDRESS);
+
+      for (int i = 0; i < 10; i++) {
+         message = session.createMessage(true);
+         message.getBodyBuffer().writeBytes(body);
+         message.putIntProperty(new SimpleString("id"), i);
+
+         // Consume messages to force an eventual out of order delivery
+         if (i == 5) {
+            ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS);
+            for (int j = 0; j < numberOfMessages; j++) {
+               ClientMessage msg = consumer.receive(PagingTest.RECEIVE_TIMEOUT);
+               msg.acknowledge();
+               Assert.assertNotNull(msg);
+            }
+
+            Assert.assertNull(consumer.receiveImmediate());
+            consumer.close();
+         }
+
+         Integer messageID = (Integer) message.getObjectProperty(new SimpleString("id"));
+         Assert.assertNotNull(messageID);
+         Assert.assertEquals(messageID.intValue(), i);
+
+         producerTransacted.send(message);
+      }
+
+      ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS);
+
+      Assert.assertNull(consumer.receiveImmediate());
+
+      sessionTransacted.commit();
+
+      sessionTransacted.close();
+
+      for (int i = 0; i < 10; i++) {
+         message = consumer.receive(PagingTest.RECEIVE_TIMEOUT);
+
+         Assert.assertNotNull(message);
+
+         Integer messageID = (Integer) message.getObjectProperty(new SimpleString("id"));
+
+         Assert.assertNotNull(messageID);
+         Assert.assertEquals("message received out of order", messageID.intValue(), i);
+
+         message.acknowledge();
+      }
+
+      Assert.assertNull(consumer.receiveImmediate());
+
+      consumer.close();
+
+      session.close();
+   }
+
+   /**
+    * - Make a destination in page mode
+    * - Add stuff to a transaction
+    * - Consume the entire destination (not in page mode any more)
+    * - Add stuff to a transaction again
+    * - Check order
+    * <br>
+    * Test under discussion at : http://community.jboss.org/thread/154061?tstart=0
+    */
+   @Test
+   public void testDepageDuringTransaction2() throws Exception {
+      boolean IS_DURABLE_MESSAGE = true;
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig();
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+      sf = createSessionFactory(locator);
+
+      byte[] body = new byte[MESSAGE_SIZE];
+
+      ClientSession sessionTransacted = sf.createSession(null, null, false, false, false, false, 0);
+      ClientProducer producerTransacted = sessionTransacted.createProducer(PagingTest.ADDRESS);
+
+      ClientSession session = sf.createSession(null, null, false, true, true, false, 0);
+      session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS, null, true);
+
+      ClientMessage firstMessage = sessionTransacted.createMessage(IS_DURABLE_MESSAGE);
+      firstMessage.getBodyBuffer().writeBytes(body);
+      firstMessage.putIntProperty(new SimpleString("id"), 0);
+
+      producerTransacted.send(firstMessage);
+
+      ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
+
+      ClientMessage message = null;
+
+      int numberOfMessages = 0;
+      while (true) {
+         System.out.println("Sending message " + numberOfMessages);
+         message = session.createMessage(IS_DURABLE_MESSAGE);
+         message.getBodyBuffer().writeBytes(body);
+         message.putIntProperty("id", numberOfMessages);
+         message.putBooleanProperty("new", false);
+
+         // Stop sending message as soon as we start paging
+         if (server.getPagingManager().getPageStore(PagingTest.ADDRESS).isPaging()) {
+            break;
+         }
+         numberOfMessages++;
+
+         producer.send(message);
+      }
+
+      Assert.assertTrue(server.getPagingManager().getPageStore(PagingTest.ADDRESS).isPaging());
+
+      session.start();
+
+      for (int i = 1; i < 10; i++) {
+         message = session.createMessage(true);
+         message.getBodyBuffer().writeBytes(body);
+         message.putIntProperty(new SimpleString("id"), i);
+
+         // Consume messages to force an eventual out of order delivery
+         if (i == 5) {
+            ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS);
+            for (int j = 0; j < numberOfMessages; j++) {
+               ClientMessage msg = consumer.receive(PagingTest.RECEIVE_TIMEOUT);
+               msg.acknowledge();
+               assertEquals(j, msg.getIntProperty("id").intValue());
+               assertFalse(msg.getBooleanProperty("new"));
+               Assert.assertNotNull(msg);
+            }
+
+            ClientMessage msgReceived = consumer.receiveImmediate();
+
+            Assert.assertNull(msgReceived);
+            consumer.close();
+         }
+
+         Integer messageID = (Integer) message.getObjectProperty(new SimpleString("id"));
+         Assert.assertNotNull(messageID);
+         Assert.assertEquals(messageID.intValue(), i);
+
+         producerTransacted.send(message);
+      }
+
+      ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS);
+
+      Assert.assertNull(consumer.receiveImmediate());
+
+      sessionTransacted.commit();
+
+      sessionTransacted.close();
+
+      for (int i = 0; i < 10; i++) {
+         message = consumer.receive(PagingTest.RECEIVE_TIMEOUT);
+
+         Assert.assertNotNull(message);
+
+         Integer messageID = (Integer) message.getObjectProperty(new SimpleString("id"));
+
+         // System.out.println(messageID);
+         Assert.assertNotNull(messageID);
+         Assert.assertEquals("message received out of order", i, messageID.intValue());
+
+         message.acknowledge();
+      }
+
+      Assert.assertNull(consumer.receiveImmediate());
+
+      consumer.close();
+
+      session.close();
+
+   }
+
+   @Test
+   public void testDepageDuringTransaction3() throws Exception {
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig();
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+      sf = createSessionFactory(locator);
+
+      byte[] body = new byte[MESSAGE_SIZE];
+
+      ClientSession sessionTransacted = sf.createSession(null, null, false, false, false, false, 0);
+      ClientProducer producerTransacted = sessionTransacted.createProducer(PagingTest.ADDRESS);
+
+      ClientSession sessionNonTX = sf.createSession(true, true, 0);
+      sessionNonTX.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS, null, true);
+
+      ClientProducer producerNonTransacted = sessionNonTX.createProducer(PagingTest.ADDRESS);
+
+      sessionNonTX.start();
+
+      for (int i = 0; i < 50; i++) {
+         ClientMessage message = sessionNonTX.createMessage(true);
+         message.getBodyBuffer().writeBytes(body);
+         message.putIntProperty(new SimpleString("id"), i);
+         message.putStringProperty(new SimpleString("tst"), new SimpleString("i=" + i));
+
+         producerTransacted.send(message);
+
+         if (i % 2 == 0) {
+            for (int j = 0; j < 20; j++) {
+               ClientMessage msgSend = sessionNonTX.createMessage(true);
+               msgSend.putStringProperty(new SimpleString("tst"), new SimpleString("i=" + i + ", j=" + j));
+               msgSend.getBodyBuffer().writeBytes(new byte[10 * 1024]);
+               producerNonTransacted.send(msgSend);
+            }
+            assertTrue(server.getPagingManager().getPageStore(PagingTest.ADDRESS).isPaging());
+         }
+         else {
+            ClientConsumer consumer = sessionNonTX.createConsumer(PagingTest.ADDRESS);
+            for (int j = 0; j < 20; j++) {
+               ClientMessage msgReceived = consumer.receive(10000);
+               assertNotNull(msgReceived);
+               msgReceived.acknowledge();
+            }
+            consumer.close();
+         }
+      }
+
+      ClientConsumer consumerNonTX = sessionNonTX.createConsumer(PagingTest.ADDRESS);
+      while (true) {
+         ClientMessage msgReceived = consumerNonTX.receive(1000);
+         if (msgReceived == null) {
+            break;
+         }
+         msgReceived.acknowledge();
+      }
+      consumerNonTX.close();
+
+      ClientConsumer consumer = sessionNonTX.createConsumer(PagingTest.ADDRESS);
+
+      Assert.assertNull(consumer.receiveImmediate());
+
+      sessionTransacted.commit();
+
+      sessionTransacted.close();
+
+      for (int i = 0; i < 50; i++) {
+         ClientMessage message = consumer.receive(PagingTest.RECEIVE_TIMEOUT);
+
+         Assert.assertNotNull(message);
+
+         Integer messageID = (Integer) message.getObjectProperty(new SimpleString("id"));
+
+         Assert.assertNotNull(messageID);
+         Assert.assertEquals("message received out of order", i, messageID.intValue());
+
+         message.acknowledge();
+      }
+
+      Assert.assertNull(consumer.receiveImmediate());
+
+      consumer.close();
+
+      sessionNonTX.close();
+   }
+
+   @Test
+   public void testDepageDuringTransaction4() throws Exception {
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false).setJournalSyncTransactional(false);
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      final AtomicInteger errors = new AtomicInteger(0);
+
+      final int numberOfMessages = 10000;
+
+      locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(false);
+
+      sf = createSessionFactory(locator);
+
+      final byte[] body = new byte[MESSAGE_SIZE];
+
+      Thread producerThread = new Thread() {
+         @Override
+         public void run() {
+            ClientSession sessionProducer = null;
+            try {
+               sessionProducer = sf.createSession(false, false);
+               ClientProducer producer = sessionProducer.createProducer(ADDRESS);
+
+               for (int i = 0; i < numberOfMessages; i++) {
+                  ClientMessage msg = sessionProducer.createMessage(true);
+                  msg.getBodyBuffer().writeBytes(body);
+                  msg.putIntProperty("count", i);
+                  producer.send(msg);
+
+                  if (i % 100 == 0 && i != 0) {
+                     sessionProducer.commit();
+                     // Thread.sleep(500);
+                  }
+               }
+
+               sessionProducer.commit();
+
+            }
+            catch (Throwable e) {
+               e.printStackTrace(); // >> junit report
+               errors.incrementAndGet();
+            }
+            finally {
+               try {
+                  if (sessionProducer != null) {
+                     sessionProducer.close();
+                  }
+               }
+               catch (Throwable e) {
+                  e.printStackTrace();
+                  errors.incrementAndGet();
+               }
+            }
+         }
+      };
+
+      ClientSession session = sf.createSession(true, true, 0);
+      session.start();
+      session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS, null, true);
+
+      producerThread.start();
+
+      ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS);
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         ClientMessage msg = consumer.receive(5000);
+         assertNotNull(msg);
+         assertEquals(i, msg.getIntProperty("count").intValue());
+         msg.acknowledge();
+         if (i > 0 && i % 10 == 0) {
+            session.commit();
+         }
+      }
+      session.commit();
+
+      session.close();
+
+      producerThread.join();
+
+      locator.close();
+
+      sf.close();
+
+      assertEquals(0, errors.get());
+   }
+
+   @Test
+   public void testOrderingNonTX() throws Exception {
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false).setJournalSyncTransactional(false);
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_SIZE * 2);
+
+      server.getConfiguration();
+      server.getConfiguration();
+
+      server.start();
+
+      final AtomicInteger errors = new AtomicInteger(0);
+
+      final int numberOfMessages = 2000;
+
+      locator.setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+      sf = createSessionFactory(locator);
+
+      final CountDownLatch ready = new CountDownLatch(1);
+
+      final byte[] body = new byte[MESSAGE_SIZE];
+
+      Thread producerThread = new Thread() {
+         @Override
+         public void run() {
+            ClientSession sessionProducer = null;
+            try {
+               sessionProducer = sf.createSession(true, true);
+               ClientProducer producer = sessionProducer.createProducer(ADDRESS);
+
+               for (int i = 0; i < numberOfMessages; i++) {
+                  ClientMessage msg = sessionProducer.createMessage(true);
+                  msg.getBodyBuffer().writeBytes(body);
+                  msg.putIntProperty("count", i);
+                  producer.send(msg);
+
+                  if (i == 1000) {
+                     // The session is not TX, but we do this just to perform a round trip to the server
+                     // and make sure there are no pending messages
+                     sessionProducer.commit();
+
+                     assertTrue(server.getPagingManager().getPageStore(ADDRESS).isPaging());
+                     ready.countDown();
+                  }
+               }
+
+               sessionProducer.commit();
+
+               log.info("Producer gone");
+
+            }
+            catch (Throwable e) {
+               e.printStackTrace(); // >> junit report
+               errors.incrementAndGet();
+            }
+            finally {
+               try {
+                  if (sessionProducer != null) {
+                     sessionProducer.close();
+                  }
+               }
+               catch (Throwable e) {
+                  e.printStackTrace();
+                  errors.incrementAndGet();
+               }
+            }
+         }
+      };
+
+      ClientSession session = sf.createSession(true, true, 0);
+      session.start();
+      session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS, null, true);
+
+      producerThread.start();
+
+      assertTrue(ready.await(100, TimeUnit.SECONDS));
+
+      ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS);
+
+      for (int i = 0; i < numberOfMessages; i++) {
+         ClientMessage msg = consumer.receive(5000);
+         assertNotNull(msg);
+         if (i != msg.getIntProperty("count").intValue()) {
+            log.info("Received " + i + " with property = " + msg.getIntProperty("count"));
+            log.info("###### different");
+         }
+         // assertEquals(i, msg.getIntProperty("count").intValue());
+         msg.acknowledge();
+      }
+
+      session.close();
+
+      producerThread.join();
+
+      assertEquals(0, errors.get());
+   }
+
+   @Test
+   public void testPageOnSchedulingNoRestart() throws Exception {
+      internalTestPageOnScheduling(false);
+   }
+
+   @Test
+   public void testPageOnSchedulingRestart() throws Exception {
+      internalTestPageOnScheduling(true);
+   }
+
+   public void internalTestPageOnScheduling(final boolean restart) throws Exception {
+      clearDataRecreateServerDirs();
+
+      Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false);
+
+      server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX);
+
+      server.start();
+
+      final int numberOfMessages = 1000;
+
+      final int numberOfBytes = 1024;
+
+      locator.setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true);
+
+      sf = createSessionFactory(locator);
+      ClientSession session = sf.createSession(null, null, false, true, true, false, 0);
+
+      session.createQueue(PagingTest.ADDRESS, PagingTest.ADDRESS, null, true);
+
+      ClientProducer producer = session.createProducer(PagingTest.ADDRESS);
+
+      ClientMessage message = null;
+
+      byte[] body = new byte[numberOfBytes];
+
+      for (int j = 0; j < numberOfBytes; j++) {
+         body[j] = ActiveMQTestBase.getSamplebyte(j);
+      }
+
+      long scheduledTime = Sys

<TRUNCATED>