You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2020/05/07 02:12:34 UTC
[james-project] 10/22: JAMES-3148 Test and correct metadata cleanup
upon failures
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 0192d7022f3775039cb6d47d9cd021bfc70f39fe
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Sun Apr 12 16:29:02 2020 +0700
JAMES-3148 Test and correct metadata cleanup upon failures
Delete the lowest levels of metadata first so that retries eventually
clean up everything.
---
.../mailbox/cassandra/DeleteMessageListener.java | 22 ++-
.../cassandra/CassandraMailboxManagerTest.java | 220 +++++++++++++++++++++
2 files changed, 239 insertions(+), 3 deletions(-)
diff --git a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/DeleteMessageListener.java b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/DeleteMessageListener.java
index 31529e6..81afcd1 100644
--- a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/DeleteMessageListener.java
+++ b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/DeleteMessageListener.java
@@ -102,9 +102,9 @@ public class DeleteMessageListener implements MailboxListener.GroupMailboxListen
messageIdDAO.retrieveMessages(mailboxId, MessageRange.all())
.map(ComposedMessageIdWithMetaData::getComposedMessageId)
- .concatMap(metadata -> imapUidDAO.delete((CassandraMessageId) metadata.getMessageId(), mailboxId)
- .then(messageIdDAO.delete(mailboxId, metadata.getUid()))
- .then(handleDeletion((CassandraMessageId) metadata.getMessageId())))
+ .concatMap(metadata -> handleDeletion((CassandraMessageId) metadata.getMessageId(), mailboxId)
+ .then(imapUidDAO.delete((CassandraMessageId) metadata.getMessageId(), mailboxId))
+ .then(messageIdDAO.delete(mailboxId, metadata.getUid())))
.then()
.block();
}
@@ -119,6 +119,15 @@ public class DeleteMessageListener implements MailboxListener.GroupMailboxListen
.then(messageDAO.delete(messageId)));
}
+ private Mono<Void> handleDeletion(CassandraMessageId messageId, CassandraId excludedId) {
+ return Mono.just(messageId)
+ .filterWhen(id -> isReferenced(id, excludedId))
+ .flatMap(id -> readMessage(id)
+ .flatMap(message -> deleteUnreferencedAttachments(message).thenReturn(message))
+ .flatMap(this::deleteAttachmentMessageIds)
+ .then(messageDAO.delete(messageId)));
+ }
+
private Mono<MessageRepresentation> readMessage(CassandraMessageId id) {
return messageDAO.retrieveMessage(id, MessageMapper.FetchType.Metadata);
}
@@ -149,4 +158,11 @@ public class DeleteMessageListener implements MailboxListener.GroupMailboxListen
.hasElements()
.map(negate());
}
+
+ private Mono<Boolean> isReferenced(CassandraMessageId id, CassandraId excludedId) {
+ return imapUidDAO.retrieve(id, ALL_MAILBOXES)
+ .filter(metadata -> !metadata.getComposedMessageId().getMailboxId().equals(excludedId))
+ .hasElements()
+ .map(negate());
+ }
}
diff --git a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/CassandraMailboxManagerTest.java b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/CassandraMailboxManagerTest.java
index 3f427a3..8d14e71 100644
--- a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/CassandraMailboxManagerTest.java
+++ b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/CassandraMailboxManagerTest.java
@@ -18,6 +18,7 @@
****************************************************************/
package org.apache.james.mailbox.cassandra;
+import static org.apache.james.backends.cassandra.Scenario.Builder.fail;
import static org.mockito.Mockito.mock;
import java.util.Collection;
@@ -135,6 +136,161 @@ public class CassandraMailboxManagerTest extends MailboxManagerTest<CassandraMai
});
}
+ @Test
+ void deleteMailboxShouldEventuallyUnreferenceMessageMetadataWhenDeleteAttachmentFails(CassandraCluster cassandraCluster) throws Exception {
+ ComposedMessageId composedMessageId = inboxManager.appendMessage(MessageManager.AppendCommand.builder()
+ .build(ClassLoaderUtils.getSystemResourceAsByteArray("eml/emailWithOnlyAttachment.eml")), session);
+
+ AttachmentId attachmentId = Iterators.toStream(inboxManager.getMessages(MessageRange.all(), FetchGroup.FULL_CONTENT, session))
+ .map(Throwing.function(MessageResult::getLoadedAttachments))
+ .flatMap(Collection::stream)
+ .map(MessageAttachment::getAttachmentId)
+ .findFirst()
+ .get();
+
+ cassandraCluster.getConf().registerScenario(fail()
+ .times(1)
+ .whenQueryStartsWith("DELETE FROM attachmentV2 WHERE idAsUUID=:idAsUUID;"));
+
+ mailboxManager.deleteMailbox(inbox, session);
+
+ SoftAssertions.assertSoftly(softly -> {
+ CassandraMessageId cassandraMessageId = (CassandraMessageId) composedMessageId.getMessageId();
+ CassandraId mailboxId = (CassandraId) composedMessageId.getMailboxId();
+
+ softly.assertThat(messageDAO(cassandraCluster).retrieveMessage(cassandraMessageId, MessageMapper.FetchType.Metadata)
+ .blockOptional()).isEmpty();
+
+ softly.assertThat(imapUidDAO(cassandraCluster).retrieve(cassandraMessageId, Optional.of(mailboxId)).collectList().block())
+ .isEmpty();
+
+ softly.assertThat(messageIdDAO(cassandraCluster).retrieveMessages(mailboxId, MessageRange.all()).collectList().block())
+ .isEmpty();
+
+ softly.assertThat(attachmentDAO(cassandraCluster).getAttachment(attachmentId).blockOptional())
+ .isEmpty();
+
+ softly.assertThat(attachmentMessageIdDAO(cassandraCluster).getOwnerMessageIds(attachmentId).collectList().block())
+ .doesNotContain(cassandraMessageId);
+ });
+ }
+
+ @Test
+ void deleteMailboxShouldEventuallyUnreferenceMessageMetadataWhenDeleteMessageFails(CassandraCluster cassandraCluster) throws Exception {
+ ComposedMessageId composedMessageId = inboxManager.appendMessage(MessageManager.AppendCommand.builder()
+ .build(ClassLoaderUtils.getSystemResourceAsByteArray("eml/emailWithOnlyAttachment.eml")), session);
+
+ AttachmentId attachmentId = Iterators.toStream(inboxManager.getMessages(MessageRange.all(), FetchGroup.FULL_CONTENT, session))
+ .map(Throwing.function(MessageResult::getLoadedAttachments))
+ .flatMap(Collection::stream)
+ .map(MessageAttachment::getAttachmentId)
+ .findFirst()
+ .get();
+
+ cassandraCluster.getConf().registerScenario(fail()
+ .times(1)
+ .whenQueryStartsWith("DELETE FROM messageV2 WHERE messageId=:messageId;"));
+
+ mailboxManager.deleteMailbox(inbox, session);
+
+ SoftAssertions.assertSoftly(softly -> {
+ CassandraMessageId cassandraMessageId = (CassandraMessageId) composedMessageId.getMessageId();
+ CassandraId mailboxId = (CassandraId) composedMessageId.getMailboxId();
+
+ softly.assertThat(messageDAO(cassandraCluster).retrieveMessage(cassandraMessageId, MessageMapper.FetchType.Metadata)
+ .blockOptional()).isEmpty();
+
+ softly.assertThat(imapUidDAO(cassandraCluster).retrieve(cassandraMessageId, Optional.of(mailboxId)).collectList().block())
+ .isEmpty();
+
+ softly.assertThat(messageIdDAO(cassandraCluster).retrieveMessages(mailboxId, MessageRange.all()).collectList().block())
+ .isEmpty();
+
+ softly.assertThat(attachmentDAO(cassandraCluster).getAttachment(attachmentId).blockOptional())
+ .isEmpty();
+
+ softly.assertThat(attachmentMessageIdDAO(cassandraCluster).getOwnerMessageIds(attachmentId).collectList().block())
+ .doesNotContain(cassandraMessageId);
+ });
+ }
+
+ @Test
+ void deleteMailboxShouldEventuallyUnreferenceMessageMetadataWhenDeleteMailboxContextFails(CassandraCluster cassandraCluster) throws Exception {
+ ComposedMessageId composedMessageId = inboxManager.appendMessage(MessageManager.AppendCommand.builder()
+ .build(ClassLoaderUtils.getSystemResourceAsByteArray("eml/emailWithOnlyAttachment.eml")), session);
+
+ AttachmentId attachmentId = Iterators.toStream(inboxManager.getMessages(MessageRange.all(), FetchGroup.FULL_CONTENT, session))
+ .map(Throwing.function(MessageResult::getLoadedAttachments))
+ .flatMap(Collection::stream)
+ .map(MessageAttachment::getAttachmentId)
+ .findFirst()
+ .get();
+
+ cassandraCluster.getConf().registerScenario(fail()
+ .times(1)
+ .whenQueryStartsWith("DELETE FROM messageIdTable"));
+
+ mailboxManager.deleteMailbox(inbox, session);
+
+ SoftAssertions.assertSoftly(softly -> {
+ CassandraMessageId cassandraMessageId = (CassandraMessageId) composedMessageId.getMessageId();
+ CassandraId mailboxId = (CassandraId) composedMessageId.getMailboxId();
+
+ softly.assertThat(messageDAO(cassandraCluster).retrieveMessage(cassandraMessageId, MessageMapper.FetchType.Metadata)
+ .blockOptional()).isEmpty();
+
+ softly.assertThat(imapUidDAO(cassandraCluster).retrieve(cassandraMessageId, Optional.of(mailboxId)).collectList().block())
+ .isEmpty();
+
+ softly.assertThat(messageIdDAO(cassandraCluster).retrieveMessages(mailboxId, MessageRange.all()).collectList().block())
+ .isEmpty();
+
+ softly.assertThat(attachmentDAO(cassandraCluster).getAttachment(attachmentId).blockOptional())
+ .isEmpty();
+
+ softly.assertThat(attachmentMessageIdDAO(cassandraCluster).getOwnerMessageIds(attachmentId).collectList().block())
+ .doesNotContain(cassandraMessageId);
+ });
+ }
+
+ @Test
+ void deleteMailboxShouldEventuallyUnreferenceMessageMetadataWhenDeleteMailboxContextByIdFails(CassandraCluster cassandraCluster) throws Exception {
+ ComposedMessageId composedMessageId = inboxManager.appendMessage(MessageManager.AppendCommand.builder()
+ .build(ClassLoaderUtils.getSystemResourceAsByteArray("eml/emailWithOnlyAttachment.eml")), session);
+
+ AttachmentId attachmentId = Iterators.toStream(inboxManager.getMessages(MessageRange.all(), FetchGroup.FULL_CONTENT, session))
+ .map(Throwing.function(MessageResult::getLoadedAttachments))
+ .flatMap(Collection::stream)
+ .map(MessageAttachment::getAttachmentId)
+ .findFirst()
+ .get();
+
+ cassandraCluster.getConf().registerScenario(fail()
+ .times(1)
+ .whenQueryStartsWith("DELETE FROM imapUidTable"));
+
+ mailboxManager.deleteMailbox(inbox, session);
+
+ SoftAssertions.assertSoftly(softly -> {
+ CassandraMessageId cassandraMessageId = (CassandraMessageId) composedMessageId.getMessageId();
+ CassandraId mailboxId = (CassandraId) composedMessageId.getMailboxId();
+
+ softly.assertThat(messageDAO(cassandraCluster).retrieveMessage(cassandraMessageId, MessageMapper.FetchType.Metadata)
+ .blockOptional()).isEmpty();
+
+ softly.assertThat(imapUidDAO(cassandraCluster).retrieve(cassandraMessageId, Optional.of(mailboxId)).collectList().block())
+ .isEmpty();
+
+ softly.assertThat(messageIdDAO(cassandraCluster).retrieveMessages(mailboxId, MessageRange.all()).collectList().block())
+ .isEmpty();
+
+ softly.assertThat(attachmentDAO(cassandraCluster).getAttachment(attachmentId).blockOptional())
+ .isEmpty();
+
+ softly.assertThat(attachmentMessageIdDAO(cassandraCluster).getOwnerMessageIds(attachmentId).collectList().block())
+ .doesNotContain(cassandraMessageId);
+ });
+ }
@Test
void deleteShouldUnreferenceMessageMetadata(CassandraCluster cassandraCluster) throws Exception {
@@ -165,6 +321,70 @@ public class CassandraMailboxManagerTest extends MailboxManagerTest<CassandraMai
}
@Test
+ void deleteShouldUnreferenceMessageMetadataWhenDeleteMessageFails(CassandraCluster cassandraCluster) throws Exception {
+ ComposedMessageId composedMessageId = inboxManager.appendMessage(MessageManager.AppendCommand.builder()
+ .build(ClassLoaderUtils.getSystemResourceAsByteArray("eml/emailWithOnlyAttachment.eml")), session);
+
+ AttachmentId attachmentId = Iterators.toStream(inboxManager.getMessages(MessageRange.all(), FetchGroup.FULL_CONTENT, session))
+ .map(Throwing.function(MessageResult::getLoadedAttachments))
+ .flatMap(Collection::stream)
+ .map(MessageAttachment::getAttachmentId)
+ .findFirst()
+ .get();
+
+ cassandraCluster.getConf().registerScenario(fail()
+ .times(1)
+ .whenQueryStartsWith("DELETE FROM messageV2 WHERE messageId=:messageId;"));
+
+ inboxManager.delete(ImmutableList.of(composedMessageId.getUid()), session);
+
+ SoftAssertions.assertSoftly(softly -> {
+ CassandraMessageId cassandraMessageId = (CassandraMessageId) composedMessageId.getMessageId();
+
+ softly.assertThat(messageDAO(cassandraCluster).retrieveMessage(cassandraMessageId, MessageMapper.FetchType.Metadata)
+ .blockOptional()).isEmpty();
+
+ softly.assertThat(attachmentDAO(cassandraCluster).getAttachment(attachmentId).blockOptional())
+ .isEmpty();
+
+ softly.assertThat(attachmentMessageIdDAO(cassandraCluster).getOwnerMessageIds(attachmentId).collectList().block())
+ .doesNotContain(cassandraMessageId);
+ });
+ }
+
+ @Test
+ void deleteShouldUnreferenceMessageMetadataWhenDeleteAttachmentFails(CassandraCluster cassandraCluster) throws Exception {
+ ComposedMessageId composedMessageId = inboxManager.appendMessage(MessageManager.AppendCommand.builder()
+ .build(ClassLoaderUtils.getSystemResourceAsByteArray("eml/emailWithOnlyAttachment.eml")), session);
+
+ AttachmentId attachmentId = Iterators.toStream(inboxManager.getMessages(MessageRange.all(), FetchGroup.FULL_CONTENT, session))
+ .map(Throwing.function(MessageResult::getLoadedAttachments))
+ .flatMap(Collection::stream)
+ .map(MessageAttachment::getAttachmentId)
+ .findFirst()
+ .get();
+
+ cassandraCluster.getConf().registerScenario(fail()
+ .times(1)
+ .whenQueryStartsWith("DELETE FROM attachmentV2 WHERE idAsUUID=:idAsUUID;"));
+
+ inboxManager.delete(ImmutableList.of(composedMessageId.getUid()), session);
+
+ SoftAssertions.assertSoftly(softly -> {
+ CassandraMessageId cassandraMessageId = (CassandraMessageId) composedMessageId.getMessageId();
+
+ softly.assertThat(messageDAO(cassandraCluster).retrieveMessage(cassandraMessageId, MessageMapper.FetchType.Metadata)
+ .blockOptional()).isEmpty();
+
+ softly.assertThat(attachmentDAO(cassandraCluster).getAttachment(attachmentId).blockOptional())
+ .isEmpty();
+
+ softly.assertThat(attachmentMessageIdDAO(cassandraCluster).getOwnerMessageIds(attachmentId).collectList().block())
+ .doesNotContain(cassandraMessageId);
+ });
+ }
+
+ @Test
void deleteMailboxShouldNotUnreferenceMessageMetadataWhenOtherReference(CassandraCluster cassandraCluster) throws Exception {
ComposedMessageId composedMessageId = inboxManager.appendMessage(MessageManager.AppendCommand.builder()
.build(ClassLoaderUtils.getSystemResourceAsByteArray("eml/emailWithOnlyAttachment.eml")), session);
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org