You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@james.apache.org by bt...@apache.org on 2023/01/04 02:31:20 UTC
[james-project] 01/03: JAMES-3872 Add a FetchType that support getting the attachments' metadata without fetching the body content
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 5cb1ed5546d4840afca1d22cd3bb011e0553465d
Author: Quan Tran <hq...@linagora.com>
AuthorDate: Wed Dec 28 18:01:03 2022 +0700
JAMES-3872 Add a FetchType that support getting the attachments' metadata without fetching the body content
---
.../org/apache/james/mailbox/model/FetchGroup.java | 2 +
.../mailbox/cassandra/mail/AttachmentLoader.java | 2 +-
.../cassandra/mail/CassandraMessageDAO.java | 1 +
.../cassandra/mail/CassandraMessageDAOV3.java | 1 +
.../cassandra/mail/CassandraMessageMapperTest.java | 47 ++++++++++++++++++++++
.../store/search/SimpleMessageSearchIndexTest.java | 2 +-
.../org/apache/james/mailbox/store/BatchSizes.java | 3 +-
.../apache/james/mailbox/store/ResultUtils.java | 3 +-
.../mailbox/store/mail/FetchGroupConverter.java | 8 +++-
.../james/mailbox/store/mail/MessageMapper.java | 9 ++++-
.../store/mail/FetchGroupConverterTest.java | 1 +
.../store/mail/model/MessageMapperTest.java | 5 ++-
12 files changed, 77 insertions(+), 7 deletions(-)
diff --git a/mailbox/api/src/main/java/org/apache/james/mailbox/model/FetchGroup.java b/mailbox/api/src/main/java/org/apache/james/mailbox/model/FetchGroup.java
index fef49c09ea..396dbf5d16 100644
--- a/mailbox/api/src/main/java/org/apache/james/mailbox/model/FetchGroup.java
+++ b/mailbox/api/src/main/java/org/apache/james/mailbox/model/FetchGroup.java
@@ -36,6 +36,7 @@ public class FetchGroup extends Profiles<FetchGroup> {
public enum Profile {
MIME_DESCRIPTOR,
HEADERS,
+ HEADERS_WITH_ATTACHMENTS_METADATA,
FULL_CONTENT,
BODY_CONTENT,
MIME_HEADERS,
@@ -48,6 +49,7 @@ public class FetchGroup extends Profiles<FetchGroup> {
*/
public static final FetchGroup MINIMAL = new FetchGroup(EnumSet.noneOf(Profile.class));
public static final FetchGroup HEADERS = new FetchGroup(EnumSet.of(Profile.HEADERS));
+ public static final FetchGroup HEADERS_WITH_ATTACHMENTS_METADATA = new FetchGroup(EnumSet.of(Profile.HEADERS_WITH_ATTACHMENTS_METADATA));
public static final FetchGroup FULL_CONTENT = new FetchGroup(EnumSet.of(Profile.FULL_CONTENT));
public static final FetchGroup BODY_CONTENT = new FetchGroup(EnumSet.of(Profile.BODY_CONTENT));
diff --git a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/AttachmentLoader.java b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/AttachmentLoader.java
index f37267caa2..c91d12b7b1 100644
--- a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/AttachmentLoader.java
+++ b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/AttachmentLoader.java
@@ -53,7 +53,7 @@ public class AttachmentLoader {
}
private Mono<List<MessageAttachmentMetadata>> loadAttachments(Stream<MessageAttachmentRepresentation> messageAttachmentRepresentations, MessageMapper.FetchType fetchType) {
- if (fetchType == MessageMapper.FetchType.FULL) {
+ if (fetchType == MessageMapper.FetchType.FULL || fetchType == MessageMapper.FetchType.ATTACHMENTS_METADATA) {
return getAttachments(messageAttachmentRepresentations.collect(ImmutableList.toImmutableList()));
} else {
return Mono.just(ImmutableList.of());
diff --git a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAO.java b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAO.java
index 040b2b29b5..6c1ecec6d1 100644
--- a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAO.java
+++ b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAO.java
@@ -324,6 +324,7 @@ public class CassandraMessageDAO {
switch (fetchType) {
case FULL:
return getFullContent(headerId, bodyId);
+ case ATTACHMENTS_METADATA:
case HEADERS:
return getContent(headerId, SIZE_BASED);
case METADATA:
diff --git a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAOV3.java b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAOV3.java
index a7bf3669c4..575c8852fd 100644
--- a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAOV3.java
+++ b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageDAOV3.java
@@ -397,6 +397,7 @@ public class CassandraMessageDAOV3 {
switch (fetchType) {
case FULL:
return getFullContent(headerId, bodyId);
+ case ATTACHMENTS_METADATA:
case HEADERS:
return getContent(headerId, SIZE_BASED)
.map(ByteContent::new);
diff --git a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageMapperTest.java b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageMapperTest.java
index a459de58ff..028d4cf0d0 100644
--- a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageMapperTest.java
+++ b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/CassandraMessageMapperTest.java
@@ -23,8 +23,12 @@ import static org.apache.james.backends.cassandra.Scenario.Builder.fail;
import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Durations.ONE_SECOND;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.time.Duration;
+import java.util.Date;
import java.util.Iterator;
+import java.util.List;
import java.util.Optional;
import javax.mail.Flags;
@@ -39,12 +43,22 @@ import org.apache.james.mailbox.MessageManager;
import org.apache.james.mailbox.cassandra.ids.CassandraId;
import org.apache.james.mailbox.cassandra.ids.CassandraMessageId;
import org.apache.james.mailbox.exception.MailboxException;
+import org.apache.james.mailbox.model.ByteContent;
+import org.apache.james.mailbox.model.Cid;
+import org.apache.james.mailbox.model.MessageAttachmentMetadata;
+import org.apache.james.mailbox.model.MessageId;
import org.apache.james.mailbox.model.MessageRange;
+import org.apache.james.mailbox.model.ParsedAttachment;
+import org.apache.james.mailbox.model.ThreadId;
import org.apache.james.mailbox.store.FlagsUpdateCalculator;
+import org.apache.james.mailbox.store.mail.MessageMapper;
import org.apache.james.mailbox.store.mail.MessageMapper.FetchType;
import org.apache.james.mailbox.store.mail.model.MailboxMessage;
import org.apache.james.mailbox.store.mail.model.MapperProvider;
+import org.apache.james.mailbox.store.mail.model.MessageAssert;
import org.apache.james.mailbox.store.mail.model.MessageMapperTest;
+import org.apache.james.mailbox.store.mail.model.impl.PropertyBuilder;
+import org.apache.james.mailbox.store.mail.model.impl.SimpleMailboxMessage;
import org.apache.james.util.streams.Limit;
import org.apache.james.utils.UpdatableTickingClock;
import org.assertj.core.api.SoftAssertions;
@@ -55,6 +69,7 @@ import org.junit.jupiter.api.extension.RegisterExtension;
import com.github.fge.lambdas.Throwing;
import com.google.common.collect.ImmutableList;
+import com.google.common.io.ByteSource;
class CassandraMessageMapperTest extends MessageMapperTest {
@RegisterExtension
@@ -418,4 +433,36 @@ class CassandraMessageMapperTest extends MessageMapperTest {
assertThat(messageMapper.getMailboxCounters(benwaInboxMailbox).getUnseen()).isEqualTo(4);
}
}
+
+ @Test
+ void messagesRetrievedUsingFetchTypeAttachmentsMetadataShouldNotHaveBodyDataLoaded() throws MailboxException, IOException {
+ saveMessages();
+ MessageMapper.FetchType fetchType = FetchType.ATTACHMENTS_METADATA;
+ MailboxMessage retrievedMessage = messageMapper.findInMailbox(benwaInboxMailbox, MessageRange.one(message1.getUid()), fetchType, 1).next();
+ MessageAssert.assertThat(retrievedMessage).isEqualToWithoutUid(message1, fetchType);
+ assertThat(retrievedMessage.getBodyContent().readAllBytes()).isEmpty();
+ }
+
+ @Test
+ void messagesRetrievedUsingFetchTypeAttachmentsMetadataShouldHaveAttachmentsMetadataLoaded() throws MailboxException {
+ MessageId messageId = mapperProvider.generateMessageId();
+ String content = "Subject: Test1 \n\nBody1\n.\n";
+ ParsedAttachment attachment1 = ParsedAttachment.builder()
+ .contentType("content")
+ .content(ByteSource.wrap("attachment".getBytes(StandardCharsets.UTF_8)))
+ .noName()
+ .cid(Cid.from("cid"))
+ .inline();
+ List<MessageAttachmentMetadata> messageAttachments = attachmentMapper.storeAttachments(ImmutableList.of(attachment1), messageId);
+ MailboxMessage message = new SimpleMailboxMessage(messageId, ThreadId.fromBaseMessageId(messageId), new Date(), content.length(), 16,
+ new ByteContent(content.getBytes()), new Flags(), new PropertyBuilder().build(), benwaInboxMailbox.getMailboxId(),
+ messageAttachments, Optional.empty());
+ messageMapper.add(benwaInboxMailbox, message);
+ message.setModSeq(messageMapper.getHighestModSeq(benwaInboxMailbox));
+
+ MessageMapper.FetchType fetchType = FetchType.ATTACHMENTS_METADATA;
+ MailboxMessage retrievedMessage = messageMapper.findInMailbox(benwaInboxMailbox, MessageRange.one(message.getUid()), fetchType, 1).next();
+
+ assertThat(retrievedMessage.getAttachments()).isEqualTo(message.getAttachments());
+ }
}
diff --git a/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndexTest.java b/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndexTest.java
index 5e9c62d7de..0e36139300 100644
--- a/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndexTest.java
+++ b/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndexTest.java
@@ -239,7 +239,7 @@ class SimpleMessageSearchIndexTest extends AbstractMessageSearchIndexTest {
@Test
public void canCompareFetchTypes() {
- assertThat(FetchType.values()).containsExactly(FetchType.METADATA, FetchType.HEADERS, FetchType.FULL);
+ assertThat(FetchType.values()).containsExactly(FetchType.METADATA, FetchType.HEADERS, FetchType.ATTACHMENTS_METADATA, FetchType.FULL);
assertThat(SimpleMessageSearchIndex.maxFetchType(FetchType.METADATA, FetchType.METADATA)).isEqualTo(FetchType.METADATA);
assertThat(SimpleMessageSearchIndex.maxFetchType(FetchType.METADATA, FetchType.HEADERS)).isEqualTo(FetchType.HEADERS);
diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/BatchSizes.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/BatchSizes.java
index fd8b5648a3..07e4b0edcf 100644
--- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/BatchSizes.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/BatchSizes.java
@@ -134,12 +134,13 @@ public class BatchSizes {
switch (fetchType) {
case METADATA:
return fetchMetadata;
+ case ATTACHMENTS_METADATA:
case HEADERS:
return fetchHeaders;
case FULL:
return fetchFull;
}
- throw new RuntimeException("Unknown fetchTpe: " + fetchType);
+ throw new RuntimeException("Unknown fetchType: " + fetchType);
}
public Optional<Integer> getCopyBatchSize() {
diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/ResultUtils.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/ResultUtils.java
index 9846b159fa..f5fee1ddae 100644
--- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/ResultUtils.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/ResultUtils.java
@@ -51,6 +51,7 @@ import com.google.common.annotations.VisibleForTesting;
public class ResultUtils {
private static final EnumSet<FetchGroup.Profile> SUPPORTED_GROUPS = EnumSet.of(
FetchGroup.Profile.HEADERS,
+ FetchGroup.Profile.HEADERS_WITH_ATTACHMENTS_METADATA,
FetchGroup.Profile.BODY_CONTENT,
FetchGroup.Profile.FULL_CONTENT,
FetchGroup.Profile.MIME_DESCRIPTOR);
@@ -205,7 +206,7 @@ public class ResultUtils {
if (profiles.contains(FetchGroup.Profile.MIME_CONTENT)) {
addMimeBodyContent(message, messageResult, mimePath);
}
- if (profiles.contains(FetchGroup.Profile.HEADERS)) {
+ if (profiles.contains(FetchGroup.Profile.HEADERS) || profiles.contains(FetchGroup.Profile.HEADERS_WITH_ATTACHMENTS_METADATA)) {
addHeaders(message, messageResult, mimePath);
}
if (profiles.contains(FetchGroup.Profile.MIME_HEADERS)) {
diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/mail/FetchGroupConverter.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/mail/FetchGroupConverter.java
index fb08d13858..dd4065cfb1 100644
--- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/mail/FetchGroupConverter.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/mail/FetchGroupConverter.java
@@ -51,13 +51,17 @@ public class FetchGroupConverter {
return reduce(fetchTypes);
}
- public static MessageMapper.FetchType reduce(Collection<MessageMapper.FetchType> fetchTypes) {
+ private static MessageMapper.FetchType reduce(Collection<MessageMapper.FetchType> fetchTypes) {
boolean full = fetchTypes.contains(MessageMapper.FetchType.FULL);
boolean headers = fetchTypes.contains(MessageMapper.FetchType.HEADERS);
+ boolean headersWithAttachmentsMetadata = fetchTypes.contains(MessageMapper.FetchType.ATTACHMENTS_METADATA);
if (full) {
return MessageMapper.FetchType.FULL;
}
+ if (headersWithAttachmentsMetadata) {
+ return MessageMapper.FetchType.ATTACHMENTS_METADATA;
+ }
if (headers) {
return MessageMapper.FetchType.HEADERS;
}
@@ -68,6 +72,8 @@ public class FetchGroupConverter {
switch (profile) {
case HEADERS:
return MessageMapper.FetchType.HEADERS;
+ case HEADERS_WITH_ATTACHMENTS_METADATA:
+ return MessageMapper.FetchType.ATTACHMENTS_METADATA;
case BODY_CONTENT:
case FULL_CONTENT:
case MIME_CONTENT:
diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/mail/MessageMapper.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/mail/MessageMapper.java
index 3d25e3507f..f37068826f 100644
--- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/mail/MessageMapper.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/mail/MessageMapper.java
@@ -307,7 +307,14 @@ public interface MessageMapper extends Mapper {
* </p>
*/
HEADERS,
-
+ /**
+ * Fetch the {@link #HEADERS}, {@link Property}'s and the {@link #ATTACHMENTS_METADATA}'s for the {@link MailboxMessage}. This includes:
+ *
+ * <p>
+ * {@link MailboxMessage#getAttachments()}
+ * </p>
+ */
+ ATTACHMENTS_METADATA,
/**
* Fetch the complete {@link MailboxMessage}
*
diff --git a/mailbox/store/src/test/java/org/apache/james/mailbox/store/mail/FetchGroupConverterTest.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/mail/FetchGroupConverterTest.java
index 9898c64ba0..d69459d8a5 100644
--- a/mailbox/store/src/test/java/org/apache/james/mailbox/store/mail/FetchGroupConverterTest.java
+++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/mail/FetchGroupConverterTest.java
@@ -39,6 +39,7 @@ class FetchGroupConverterTest {
return Stream.of(
Arguments.arguments(FetchGroup.MINIMAL, FetchType.METADATA),
Arguments.arguments(FetchGroup.HEADERS, FetchType.HEADERS),
+ Arguments.arguments(FetchGroup.HEADERS_WITH_ATTACHMENTS_METADATA, FetchType.ATTACHMENTS_METADATA),
Arguments.arguments(FetchGroup.BODY_CONTENT, FetchType.FULL),
Arguments.arguments(FetchGroup.FULL_CONTENT, FetchType.FULL),
Arguments.arguments(FetchGroup.BODY_CONTENT.with(Profile.HEADERS), FetchType.FULL),
diff --git a/mailbox/store/src/test/java/org/apache/james/mailbox/store/mail/model/MessageMapperTest.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/mail/model/MessageMapperTest.java
index f2b478d744..2a49fc1af6 100644
--- a/mailbox/store/src/test/java/org/apache/james/mailbox/store/mail/model/MessageMapperTest.java
+++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/mail/model/MessageMapperTest.java
@@ -53,6 +53,7 @@ import org.apache.james.mailbox.model.ThreadId;
import org.apache.james.mailbox.model.UidValidity;
import org.apache.james.mailbox.model.UpdatedFlags;
import org.apache.james.mailbox.store.FlagsUpdateCalculator;
+import org.apache.james.mailbox.store.mail.AttachmentMapper;
import org.apache.james.mailbox.store.mail.MailboxMapper;
import org.apache.james.mailbox.store.mail.MessageMapper;
import org.apache.james.mailbox.store.mail.MessageMapper.FetchType;
@@ -81,9 +82,10 @@ public abstract class MessageMapperTest {
protected static final String USER_FLAG = "userFlag";
- private MapperProvider mapperProvider;
+ protected MapperProvider mapperProvider;
protected MessageMapper messageMapper;
private MailboxMapper mailboxMapper;
+ protected AttachmentMapper attachmentMapper;
protected Mailbox benwaInboxMailbox;
protected Mailbox benwaWorkMailbox;
@@ -107,6 +109,7 @@ public abstract class MessageMapperTest {
this.messageMapper = mapperProvider.createMessageMapper();
this.mailboxMapper = mapperProvider.createMailboxMapper();
+ this.attachmentMapper = mapperProvider.createAttachmentMapper();
initData();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org