You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2017/09/19 02:00:20 UTC
[2/6] james-project git commit: MAILBOX-306 New migration which
duplicate attachment id from message to AttachmentMessageId table
MAILBOX-306 New migration which duplicate attachment id from message to AttachmentMessageId table
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/1e808fa5
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/1e808fa5
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/1e808fa5
Branch: refs/heads/master
Commit: 1e808fa54d5452912a94549da8d2c4ab8c442ca3
Parents: 5e28f4a
Author: Raphael Ouazana <ra...@linagora.com>
Authored: Tue Sep 12 18:02:54 2017 +0200
Committer: benwa <bt...@linagora.com>
Committed: Tue Sep 19 08:59:34 2017 +0700
----------------------------------------------------------------------
.../versions/CassandraSchemaVersionManager.java | 2 +-
.../migration/AttachmentMessageIdCreation.java | 66 +++++++
.../AttachmentMessageIdCreationTest.java | 197 +++++++++++++++++++
3 files changed, 264 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/1e808fa5/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/versions/CassandraSchemaVersionManager.java
----------------------------------------------------------------------
diff --git a/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/versions/CassandraSchemaVersionManager.java b/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/versions/CassandraSchemaVersionManager.java
index 663db63..42502e8 100644
--- a/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/versions/CassandraSchemaVersionManager.java
+++ b/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/versions/CassandraSchemaVersionManager.java
@@ -34,7 +34,7 @@ import com.google.common.base.Preconditions;
public class CassandraSchemaVersionManager {
public static final int MIN_VERSION = 2;
- public static final int MAX_VERSION = 4;
+ public static final int MAX_VERSION = 5;
public static final int DEFAULT_VERSION = 2;
private static final Logger LOGGER = LoggerFactory.getLogger(CassandraSchemaVersionManager.class);
http://git-wip-us.apache.org/repos/asf/james-project/blob/1e808fa5/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/migration/AttachmentMessageIdCreation.java
----------------------------------------------------------------------
diff --git a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/migration/AttachmentMessageIdCreation.java b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/migration/AttachmentMessageIdCreation.java
new file mode 100644
index 0000000..adcc1ff
--- /dev/null
+++ b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/migration/AttachmentMessageIdCreation.java
@@ -0,0 +1,66 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.mailbox.cassandra.mail.migration;
+
+import javax.inject.Inject;
+
+import org.apache.james.mailbox.cassandra.mail.CassandraAttachmentMessageIdDAO;
+import org.apache.james.mailbox.cassandra.mail.CassandraMessageDAO;
+import org.apache.james.mailbox.cassandra.mail.CassandraMessageDAO.MessageIdAttachmentIds;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AttachmentMessageIdCreation implements Migration {
+ private static final Logger LOGGER = LoggerFactory.getLogger(AttachmentMessageIdCreation.class);
+ private final CassandraMessageDAO cassandraMessageDAO;
+ private final CassandraAttachmentMessageIdDAO attachmentMessageIdDAO;
+
+ @Inject
+ public AttachmentMessageIdCreation(CassandraMessageDAO cassandraMessageDAO,
+ CassandraAttachmentMessageIdDAO attachmentMessageIdDAO) {
+ this.cassandraMessageDAO = cassandraMessageDAO;
+ this.attachmentMessageIdDAO = attachmentMessageIdDAO;
+ }
+
+ @Override
+ public MigrationResult run() {
+ try {
+ return cassandraMessageDAO.retrieveAllMessageIdAttachmentIds()
+ .join()
+ .map(this::createIndex)
+ .reduce(MigrationResult.COMPLETED, Migration::combine);
+ } catch (Exception e) {
+ LOGGER.error("Error while creation attachmentId -> messageIds index", e);
+ return MigrationResult.PARTIAL;
+ }
+ }
+
+ private MigrationResult createIndex(MessageIdAttachmentIds message) {
+ try {
+ message.getAttachmentId()
+ .stream()
+ .forEach(attachmentId -> attachmentMessageIdDAO.storeAttachmentForMessageId(attachmentId, message.getMessageId()).join());
+ return MigrationResult.COMPLETED;
+ } catch (Exception e) {
+ LOGGER.error("Error while creation attachmentId -> messageIds index", e);
+ return MigrationResult.PARTIAL;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/james-project/blob/1e808fa5/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/migration/AttachmentMessageIdCreationTest.java
----------------------------------------------------------------------
diff --git a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/migration/AttachmentMessageIdCreationTest.java b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/migration/AttachmentMessageIdCreationTest.java
new file mode 100644
index 0000000..277741c
--- /dev/null
+++ b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/migration/AttachmentMessageIdCreationTest.java
@@ -0,0 +1,197 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.mailbox.cassandra.mail.migration;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Stream;
+
+import javax.mail.Flags;
+import javax.mail.util.SharedByteArrayInputStream;
+
+import org.apache.james.backends.cassandra.CassandraCluster;
+import org.apache.james.backends.cassandra.DockerCassandraRule;
+import org.apache.james.backends.cassandra.init.CassandraModuleComposite;
+import org.apache.james.backends.cassandra.utils.CassandraUtils;
+import org.apache.james.mailbox.MessageUid;
+import org.apache.james.mailbox.cassandra.ids.CassandraId;
+import org.apache.james.mailbox.cassandra.ids.CassandraMessageId;
+import org.apache.james.mailbox.cassandra.mail.CassandraAttachmentMessageIdDAO;
+import org.apache.james.mailbox.cassandra.mail.CassandraBlobsDAO;
+import org.apache.james.mailbox.cassandra.mail.CassandraMessageDAO;
+import org.apache.james.mailbox.cassandra.modules.CassandraAttachmentModule;
+import org.apache.james.mailbox.cassandra.modules.CassandraBlobModule;
+import org.apache.james.mailbox.cassandra.modules.CassandraMessageModule;
+import org.apache.james.mailbox.model.Attachment;
+import org.apache.james.mailbox.model.AttachmentId;
+import org.apache.james.mailbox.model.MessageAttachment;
+import org.apache.james.mailbox.model.MessageId;
+import org.apache.james.mailbox.store.mail.model.impl.PropertyBuilder;
+import org.apache.james.mailbox.store.mail.model.impl.SimpleMailboxMessage;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+public class AttachmentMessageIdCreationTest {
+ @ClassRule
+ public static DockerCassandraRule cassandraServer = new DockerCassandraRule();
+
+ private CassandraBlobsDAO blobsDAO;
+ private CassandraMessageDAO cassandraMessageDAO;
+ private CassandraAttachmentMessageIdDAO attachmentMessageIdDAO;
+
+ private AttachmentMessageIdCreation migration;
+
+ private SimpleMailboxMessage message;
+ private CassandraMessageId messageId;
+
+ @Before
+ public void setUp() {
+ CassandraCluster cassandra = CassandraCluster.create(
+ new CassandraModuleComposite(
+ new CassandraMessageModule(),
+ new CassandraAttachmentModule(),
+ new CassandraBlobModule()),
+ cassandraServer.getIp(),
+ cassandraServer.getBindingPort());
+ CassandraMessageId.Factory messageIdFactory = new CassandraMessageId.Factory();
+
+ blobsDAO = new CassandraBlobsDAO(cassandra.getConf());
+ cassandraMessageDAO = new CassandraMessageDAO(cassandra.getConf(), cassandra.getTypesProvider(),
+ blobsDAO, CassandraUtils.WITH_DEFAULT_CONFIGURATION, messageIdFactory);
+
+ attachmentMessageIdDAO = new CassandraAttachmentMessageIdDAO(cassandra.getConf(),
+ new CassandraMessageId.Factory(), CassandraUtils.WITH_DEFAULT_CONFIGURATION);
+
+ migration = new AttachmentMessageIdCreation(cassandraMessageDAO, attachmentMessageIdDAO);
+
+ messageId = messageIdFactory.generate();
+ }
+
+ @Test
+ public void emptyMigrationShouldSucceed() {
+ assertThat(migration.run())
+ .isEqualTo(Migration.MigrationResult.COMPLETED);
+ }
+
+ @Test
+ public void migrationShouldSucceedWhenNoAttachment() throws Exception {
+ List<MessageAttachment> noAttachment = ImmutableList.of();
+ message = createMessage(messageId, noAttachment);
+
+ cassandraMessageDAO.save(message).join();
+
+ assertThat(migration.run())
+ .isEqualTo(Migration.MigrationResult.COMPLETED);
+ }
+
+ @Test
+ public void migrationShouldSucceedWhenAttachment() throws Exception {
+ MessageAttachment attachment = createAttachment();
+ message = createMessage(messageId, ImmutableList.of(attachment));
+
+ cassandraMessageDAO.save(message).join();
+
+ assertThat(migration.run())
+ .isEqualTo(Migration.MigrationResult.COMPLETED);
+ }
+
+ @Test
+ public void migrationShouldCreateAttachmentIdOnAttachmentMessageIdTableFromMessage() throws Exception {
+ MessageAttachment attachment = createAttachment();
+ message = createMessage(messageId, ImmutableList.of(attachment));
+
+ cassandraMessageDAO.save(message).join();
+
+ migration.run();
+
+ assertThat(attachmentMessageIdDAO.getOwnerMessageIds(attachment.getAttachmentId()).join())
+ .containsExactly(messageId);
+ }
+
+ @Test
+ public void migrationShouldReturnPartialWhenRetrieveAllAttachmentIdFromMessageFail() throws Exception {
+ CassandraMessageDAO cassandraMessageDAO = mock(CassandraMessageDAO.class);
+ CassandraAttachmentMessageIdDAO attachmentMessageIdDAO = mock(CassandraAttachmentMessageIdDAO.class);
+ migration = new AttachmentMessageIdCreation(cassandraMessageDAO, attachmentMessageIdDAO);
+
+ when(cassandraMessageDAO.retrieveAllMessageIdAttachmentIds()).thenThrow(new RuntimeException());
+
+ assertThat(migration.run()).isEqualTo(Migration.MigrationResult.PARTIAL);
+ }
+
+ @Test
+ public void migrationShouldReturnPartialWhenSavingAttachmentIdForMessageIdFail() throws Exception {
+ CassandraMessageDAO cassandraMessageDAO = mock(CassandraMessageDAO.class);
+ CassandraAttachmentMessageIdDAO attachmentMessageIdDAO = mock(CassandraAttachmentMessageIdDAO.class);
+ CassandraMessageDAO.MessageIdAttachmentIds messageIdAttachmentIds = mock(CassandraMessageDAO.MessageIdAttachmentIds.class);
+
+ migration = new AttachmentMessageIdCreation(cassandraMessageDAO, attachmentMessageIdDAO);
+
+ when(messageIdAttachmentIds.getAttachmentId()).thenReturn(ImmutableSet.of(AttachmentId.from("any")));
+ when(cassandraMessageDAO.retrieveAllMessageIdAttachmentIds())
+ .thenReturn(CompletableFuture.completedFuture(Stream.of(messageIdAttachmentIds)));
+ when(attachmentMessageIdDAO.storeAttachmentForMessageId(any(AttachmentId.class), any(MessageId.class)))
+ .thenThrow(RuntimeException.class);
+
+ assertThat(migration.run()).isEqualTo(Migration.MigrationResult.PARTIAL);
+ }
+
+ private SimpleMailboxMessage createMessage(MessageId messageId, Collection<MessageAttachment> attachments) {
+ MessageUid messageUid = MessageUid.of(1);
+ CassandraId mailboxId = CassandraId.timeBased();
+ String content = "Subject: Any subject \n\nThis is the body\n.\n";
+ int BODY_START = 22;
+
+ return SimpleMailboxMessage.builder()
+ .messageId(messageId)
+ .mailboxId(mailboxId)
+ .uid(messageUid)
+ .internalDate(new Date())
+ .bodyStartOctet(BODY_START)
+ .size(content.length())
+ .content(new SharedByteArrayInputStream(content.getBytes(Charsets.UTF_8)))
+ .flags(new Flags())
+ .propertyBuilder(new PropertyBuilder())
+ .addAttachments(attachments)
+ .build();
+ }
+
+ private MessageAttachment createAttachment() {
+ return MessageAttachment.builder()
+ .attachment(Attachment.builder()
+ .bytes("content".getBytes(StandardCharsets.UTF_8))
+ .type("type")
+ .build())
+ .build();
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org