You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by ro...@apache.org on 2017/01/11 16:45:47 UTC
[20/22] james-project git commit: JAMES-1894 Rely on MessageID
Capability to index MessageId
JAMES-1894 Rely on MessageID Capability to index MessageId
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/941a7039
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/941a7039
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/941a7039
Branch: refs/heads/master
Commit: 941a70391b1ad1055f2ca89ae5f306e6cf79330d
Parents: 0606d2e
Author: Benoit Tellier <bt...@linagora.com>
Authored: Fri Dec 23 19:24:18 2016 +0700
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Mon Jan 9 22:01:07 2017 +0700
----------------------------------------------------------------------
...lasticSearchListeningMessageSearchIndex.java | 1 +
.../elasticsearch/json/IndexableMessage.java | 53 ++++++++++----
.../json/IndexableMessageWithMessageId.java | 72 ++++++++++++++++++++
.../json/MessageToElasticSearchJson.java | 27 ++++++--
.../search/ElasticSearchSearcher.java | 24 +++++--
.../ElasticSearchIntegrationTest.java | 3 +-
.../json/IndexableMessageTest.java | 23 +++++++
.../MailboxMessageToElasticSearchJsonTest.java | 43 ++++++++----
.../lucene/search/LuceneMessageSearchIndex.java | 45 ++++++++----
.../META-INF/spring/mailbox-index-lucene.xml | 1 +
.../LuceneMailboxMessageSearchIndexTest.java | 3 +-
.../search/LuceneMessageSearchIndexTest.java | 2 +-
.../META-INF/spring/mailbox-index-lucene.xml | 2 +
.../store/search/MessageSearchIndex.java | 13 +++-
.../mailbox/store/search/MessageSearches.java | 3 +-
.../store/search/SimpleMessageSearchIndex.java | 2 +-
.../store/src/test/resources/eml/htmlMail.json | 2 +-
mailbox/store/src/test/resources/eml/mail.json | 2 +-
.../src/test/resources/eml/nonTextual.json | 2 +-
.../src/test/resources/eml/pgpSignedMail.json | 2 +-
.../src/test/resources/eml/recursiveMail.json | 2 +-
.../eml/recursiveMailWithoutAttachments.json | 2 +-
.../store/src/test/resources/eml/spamMail.json | 2 +-
.../host/ElasticSearchHostSystem.java | 4 +-
.../host/LuceneSearchHostSystem.java | 7 +-
25 files changed, 274 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndex.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndex.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndex.java
index 281f429..850dd4c 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndex.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndex.java
@@ -99,6 +99,7 @@ public class ElasticSearchListeningMessageSearchIndex extends ListeningMessageSe
Preconditions.checkArgument(session != null, "'session' is mandatory");
return searcher.search(ImmutableList.of(session.getUser()), searchQuery, Optional.of(limit))
.map(SearchResult::getMessageId)
+ .map(com.google.common.base.Optional::get)
.collect(Guavate.toImmutableList());
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java
index dec8b8e..770f558 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java
@@ -29,11 +29,9 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.james.mailbox.MailboxSession.User;
-import org.apache.james.mailbox.MessageUid;
import org.apache.james.mailbox.elasticsearch.IndexAttachments;
import org.apache.james.mailbox.elasticsearch.query.DateResolutionFormater;
import org.apache.james.mailbox.extractor.TextExtractor;
-import org.apache.james.mailbox.model.MessageId;
import org.apache.james.mailbox.store.mail.model.MailboxMessage;
import org.apache.james.mailbox.store.mail.model.Property;
import org.apache.james.mime4j.MimeException;
@@ -48,8 +46,8 @@ import com.google.common.collect.Multimap;
public class IndexableMessage {
- public static IndexableMessage from(MailboxMessage message, List<User> users, TextExtractor textExtractor,
- ZoneId zoneId, IndexAttachments indexAttachments) {
+ public static IndexableMessage from(MailboxMessage message, List<User> users, TextExtractor textExtractor,
+ ZoneId zoneId, IndexAttachments indexAttachments) {
Preconditions.checkNotNull(message.getMailboxId());
Preconditions.checkArgument(!users.isEmpty());
@@ -90,8 +88,7 @@ public class IndexableMessage {
}
private void copyMessageFields(MailboxMessage message, ZoneId zoneId) {
- this.messageId = message.getMessageId();
- this.uid = message.getUid();
+ this.uid = message.getUid().asLong();
this.mailboxId = message.getMailboxId().serialize();
this.modSeq = message.getModSeq();
this.size = message.getFullContentOctets();
@@ -129,8 +126,7 @@ public class IndexableMessage {
.collect(Collectors.joining(" "));
}
- private MessageId messageId;
- private MessageUid uid;
+ private long uid;
private String mailboxId;
private List<String> users;
private long modSeq;
@@ -159,14 +155,47 @@ public class IndexableMessage {
private Optional<String> bodyHtml;
private String text;
- @JsonProperty(JsonMessageConstants.MESSAGE_ID)
- public String getId() {
- return messageId.serialize();
+ public IndexableMessage(long uid, String mailboxId, List<String> users, long modSeq, long size, String date, String mediaType,
+ String subType, boolean isUnRead, boolean isRecent, boolean isFlagged, boolean isDeleted, boolean isDraft,
+ boolean isAnswered, String[] userFlags, Multimap<String, String> headers, EMailers from, EMailers to,
+ EMailers cc, EMailers bcc, EMailers replyTo, Subjects subjects, String sentDate, List<Property> properties,
+ List<MimePart> attachments, Optional<String> bodyText, Optional<String> bodyHtml, String text) {
+ this.uid = uid;
+ this.mailboxId = mailboxId;
+ this.users = users;
+ this.modSeq = modSeq;
+ this.size = size;
+ this.date = date;
+ this.mediaType = mediaType;
+ this.subType = subType;
+ this.isUnRead = isUnRead;
+ this.isRecent = isRecent;
+ this.isFlagged = isFlagged;
+ this.isDeleted = isDeleted;
+ this.isDraft = isDraft;
+ this.isAnswered = isAnswered;
+ this.userFlags = userFlags;
+ this.headers = headers;
+ this.from = from;
+ this.to = to;
+ this.cc = cc;
+ this.bcc = bcc;
+ this.replyTo = replyTo;
+ this.subjects = subjects;
+ this.sentDate = sentDate;
+ this.properties = properties;
+ this.attachments = attachments;
+ this.bodyText = bodyText;
+ this.bodyHtml = bodyHtml;
+ this.text = text;
+ }
+
+ public IndexableMessage() {
}
@JsonProperty(JsonMessageConstants.UID)
public Long getUid() {
- return uid.asLong();
+ return uid;
}
@JsonProperty(JsonMessageConstants.MAILBOX_ID)
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageWithMessageId.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageWithMessageId.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageWithMessageId.java
new file mode 100644
index 0000000..8f376dc
--- /dev/null
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageWithMessageId.java
@@ -0,0 +1,72 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.mailbox.elasticsearch.json;
+
+import java.time.ZoneId;
+import java.util.List;
+import java.util.Optional;
+
+import org.apache.james.mailbox.MailboxSession;
+import org.apache.james.mailbox.elasticsearch.IndexAttachments;
+import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.store.mail.model.MailboxMessage;
+import org.apache.james.mailbox.store.mail.model.Property;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Multimap;
+
+public class IndexableMessageWithMessageId extends IndexableMessage {
+
+ public static IndexableMessage from(MailboxMessage message, List<MailboxSession.User> users, TextExtractor textExtractor,
+ ZoneId zoneId, IndexAttachments indexAttachments) {
+ IndexableMessage indexableMessage = IndexableMessage.from(message, users, textExtractor, zoneId, indexAttachments);
+ return new IndexableMessageWithMessageId(indexableMessage.getUid(), indexableMessage.getMailboxId(), indexableMessage.getUsers(),
+ indexableMessage.getModSeq(), indexableMessage.getSize(), indexableMessage.getDate(), indexableMessage.getMediaType(),
+ indexableMessage.getSubType(), indexableMessage.isUnRead(), indexableMessage.isRecent(), indexableMessage.isFlagged(),
+ indexableMessage.isDeleted(), indexableMessage.isDraft(), indexableMessage.isAnswered(), indexableMessage.getUserFlags(),
+ indexableMessage.getHeaders(), indexableMessage.getFrom(), indexableMessage.getTo(), indexableMessage.getCc(), indexableMessage.getBcc(),
+ indexableMessage.getReplyTo(), indexableMessage.getSubjects(), indexableMessage.getSentDate(), indexableMessage.getProperties(),
+ indexableMessage.getAttachments(), indexableMessage.getBodyText(), indexableMessage.getBodyHtml(), indexableMessage.getText(),
+ message.getMessageId().serialize());
+ }
+
+ private String messageId;
+
+ public IndexableMessageWithMessageId(long uid, String mailboxId, List<String> users, long modSeq, long size, String date,
+ String mediaType, String subType, boolean isUnRead, boolean isRecent, boolean isFlagged,
+ boolean isDeleted, boolean isDraft, boolean isAnswered, String[] userFlags, Multimap<String, String> headers,
+ EMailers from, EMailers to, EMailers cc, EMailers bcc, EMailers replyTo, Subjects subjects,
+ String sentDate, List<Property> properties, List<MimePart> attachments, Optional<String> bodyText,
+ Optional<String> bodyHtml, String text, String messageId) {
+ super(uid, mailboxId, users, modSeq, size, date, mediaType, subType, isUnRead, isRecent, isFlagged, isDeleted,
+ isDraft, isAnswered, userFlags, headers, from, to, cc, bcc, replyTo, subjects, sentDate, properties, attachments,
+ bodyText, bodyHtml, text);
+ this.messageId = messageId;
+ }
+
+ public IndexableMessageWithMessageId(String messageId) {
+ this.messageId = messageId;
+ }
+
+ @JsonProperty(JsonMessageConstants.MESSAGE_ID)
+ public String getMessageId() {
+ return messageId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJson.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJson.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJson.java
index b6d97e9..be23f98 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJson.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJson.java
@@ -25,10 +25,13 @@ import java.util.List;
import javax.inject.Inject;
import javax.mail.Flags;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.james.mailbox.MailboxManager;
import org.apache.james.mailbox.MailboxSession.User;
import org.apache.james.mailbox.elasticsearch.IndexAttachments;
import org.apache.james.mailbox.extractor.TextExtractor;
import org.apache.james.mailbox.store.mail.model.MailboxMessage;
+import org.apache.james.mailbox.store.search.MessageSearchIndex;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -42,24 +45,40 @@ public class MessageToElasticSearchJson {
private final TextExtractor textExtractor;
private final ZoneId zoneId;
private final IndexAttachments indexAttachments;
+ private final MessageSearchIndex.IndexMessageId indexMessageId;
- public MessageToElasticSearchJson(TextExtractor textExtractor, ZoneId zoneId, IndexAttachments indexAttachments) {
+ public MessageToElasticSearchJson(TextExtractor textExtractor, ZoneId zoneId, IndexAttachments indexAttachments, MessageSearchIndex.IndexMessageId indexMessageId) {
this.textExtractor = textExtractor;
this.zoneId = zoneId;
this.indexAttachments = indexAttachments;
this.mapper = new ObjectMapper();
this.mapper.registerModule(new GuavaModule());
this.mapper.registerModule(new Jdk8Module());
+ this.indexMessageId = indexMessageId;
}
@Inject
- public MessageToElasticSearchJson(TextExtractor textExtractor, IndexAttachments indexAttachments) {
- this(textExtractor, ZoneId.systemDefault(), indexAttachments);
+ public MessageToElasticSearchJson(TextExtractor textExtractor, IndexAttachments indexAttachments, MailboxManager mailboxManager) {
+ this(textExtractor, ZoneId.systemDefault(), indexAttachments, indexMessageId(mailboxManager));
+ }
+
+ private static MessageSearchIndex.IndexMessageId indexMessageId(MailboxManager mailboxManager) {
+ if (mailboxManager.getSupportedMessageCapabilities().contains(MailboxManager.MessageCapabilities.UniqueID)) {
+ return MessageSearchIndex.IndexMessageId.Required;
+ }
+ return MessageSearchIndex.IndexMessageId.Optional;
}
public String convertToJson(MailboxMessage message, List<User> users) throws JsonProcessingException {
Preconditions.checkNotNull(message);
- return mapper.writeValueAsString(IndexableMessage.from(message, users, textExtractor, zoneId, indexAttachments));
+ switch (indexMessageId) {
+ case Required:
+ return mapper.writeValueAsString(IndexableMessageWithMessageId.from(message, users, textExtractor, zoneId, indexAttachments));
+ case Optional:
+ return mapper.writeValueAsString(IndexableMessage.from(message, users, textExtractor, zoneId, indexAttachments));
+ default:
+ throw new NotImplementedException();
+ }
}
public String getUpdatedJsonMessagePart(Flags flags, long modSeq) throws JsonProcessingException {
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java
index 20ab38d..61bca7e 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java
@@ -34,11 +34,9 @@ import org.apache.james.mailbox.elasticsearch.query.QueryConverter;
import org.apache.james.mailbox.elasticsearch.query.SortConverter;
import org.apache.james.mailbox.exception.MailboxException;
import org.apache.james.mailbox.model.MailboxId;
-import org.apache.james.mailbox.model.MailboxId.Factory;
import org.apache.james.mailbox.model.MessageId;
import org.apache.james.mailbox.model.MultimailboxesSearchQuery;
import org.apache.james.mailbox.store.search.MessageSearchIndex;
-import org.apache.james.mailbox.store.search.SimpleMessageSearchIndex;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
@@ -57,7 +55,7 @@ public class ElasticSearchSearcher {
private final Client client;
private final QueryConverter queryConverter;
private final int size;
- private final Factory mailboxIdFactory;
+ private final MailboxId.Factory mailboxIdFactory;
private final MessageId.Factory messageIdFactory;
@Inject
@@ -65,7 +63,7 @@ public class ElasticSearchSearcher {
this(client, queryConverter, DEFAULT_SIZE, mailboxIdFactory, messageIdFactory);
}
- public ElasticSearchSearcher(Client client, QueryConverter queryConverter, int size, Factory mailboxIdFactory, MessageId.Factory messageIdFactory) {
+ public ElasticSearchSearcher(Client client, QueryConverter queryConverter, int size, MailboxId.Factory mailboxIdFactory, MessageId.Factory messageIdFactory) {
this.client = client;
this.queryConverter = queryConverter;
this.size = size;
@@ -109,11 +107,11 @@ public class ElasticSearchSearcher {
private Optional<MessageSearchIndex.SearchResult> extractContentFromHit(SearchHit hit) {
SearchHitField mailboxId = hit.field(JsonMessageConstants.MAILBOX_ID);
SearchHitField uid = hit.field(JsonMessageConstants.UID);
- SearchHitField id = hit.field(JsonMessageConstants.ID);
- if (mailboxId != null && uid != null && id != null) {
+ Optional<SearchHitField> id = retrieveMessageIdField(hit);
+ if (mailboxId != null && uid != null) {
Number uidAsNumber = uid.getValue();
return Optional.of(
- new MessageSearchIndex.SearchResult(messageIdFactory.fromString(id.getValue()),
+ new MessageSearchIndex.SearchResult(toGuava(id.map(field -> messageIdFactory.fromString(field.getValue()))),
mailboxIdFactory.fromString(mailboxId.getValue()),
MessageUid.of(uidAsNumber.longValue())));
} else {
@@ -122,4 +120,16 @@ public class ElasticSearchSearcher {
}
}
+ private Optional<SearchHitField> retrieveMessageIdField(SearchHit hit) {
+ if (hit.fields().keySet().contains(JsonMessageConstants.MESSAGE_ID)) {
+ return Optional.ofNullable(hit.field(JsonMessageConstants.MESSAGE_ID));
+ } else {
+ return Optional.empty();
+ }
+ }
+
+ private <T> com.google.common.base.Optional<T> toGuava(Optional<T> optional) {
+ return com.google.common.base.Optional.fromNullable(optional.orElse(null));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java
index 749f11b..fb73266 100644
--- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java
+++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java
@@ -40,6 +40,7 @@ import org.apache.james.mailbox.store.MailboxSessionMapperFactory;
import org.apache.james.mailbox.store.extractor.DefaultTextExtractor;
import org.apache.james.mailbox.store.mail.model.impl.MessageParser;
import org.apache.james.mailbox.store.search.AbstractMessageSearchIndexTest;
+import org.apache.james.mailbox.store.search.MessageSearchIndex;
import org.elasticsearch.client.Client;
import org.junit.Rule;
import org.junit.rules.RuleChain;
@@ -71,7 +72,7 @@ public class ElasticSearchIntegrationTest extends AbstractMessageSearchIndexTest
messageSearchIndex = new ElasticSearchListeningMessageSearchIndex(mapperFactory,
new ElasticSearchIndexer(client, new DeleteByQueryPerformer(client, Executors.newSingleThreadExecutor(), BATCH_SIZE)),
new ElasticSearchSearcher(client, new QueryConverter(new CriterionConverter()), SEARCH_SIZE, new InMemoryId.Factory(), messageIdFactory),
- new MessageToElasticSearchJson(new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.YES));
+ new MessageToElasticSearchJson(new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.YES, MessageSearchIndex.IndexMessageId.Required));
storeMailboxManager = new InMemoryMailboxManager(
mapperFactory,
new FakeAuthenticator(),
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java
index 1318c74..55ce95c 100644
--- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java
+++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java
@@ -29,6 +29,7 @@ import java.time.ZoneId;
import javax.mail.Flags;
import org.apache.commons.io.IOUtils;
+import org.apache.james.mailbox.MessageUid;
import org.apache.james.mailbox.elasticsearch.IndexAttachments;
import org.apache.james.mailbox.mock.MockMailboxSession;
import org.apache.james.mailbox.model.TestId;
@@ -40,6 +41,8 @@ import com.google.common.collect.ImmutableList;
public class IndexableMessageTest {
+ public static final MessageUid MESSAGE_UID = MessageUid.of(154);
+
@Test
public void textShouldBeEmptyWhenNoMatchingHeaders() throws Exception {
MailboxMessage mailboxMessage = mock(MailboxMessage.class);
@@ -50,6 +53,8 @@ public class IndexableMessageTest {
.thenReturn(new ByteArrayInputStream("".getBytes()));
when(mailboxMessage.createFlags())
.thenReturn(new Flags());
+ when(mailboxMessage.getUid())
+ .thenReturn(MESSAGE_UID);
IndexableMessage indexableMessage = IndexableMessage.from(mailboxMessage, ImmutableList.of(new MockMailboxSession("username").getUser()),
new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.NO);
@@ -67,6 +72,8 @@ public class IndexableMessageTest {
.thenReturn(new ByteArrayInputStream("From: First user <us...@james.org>\nFrom: Second user <us...@james.org>".getBytes()));
when(mailboxMessage.createFlags())
.thenReturn(new Flags());
+ when(mailboxMessage.getUid())
+ .thenReturn(MESSAGE_UID);
IndexableMessage indexableMessage = IndexableMessage.from(mailboxMessage, ImmutableList.of(new MockMailboxSession("username").getUser()),
new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.NO);
@@ -84,6 +91,8 @@ public class IndexableMessageTest {
.thenReturn(new ByteArrayInputStream("To: First to <us...@james.org>\nTo: Second to <us...@james.org>".getBytes()));
when(mailboxMessage.createFlags())
.thenReturn(new Flags());
+ when(mailboxMessage.getUid())
+ .thenReturn(MESSAGE_UID);
IndexableMessage indexableMessage = IndexableMessage.from(mailboxMessage, ImmutableList.of(new MockMailboxSession("username").getUser()),
new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.NO);
@@ -101,6 +110,8 @@ public class IndexableMessageTest {
.thenReturn(new ByteArrayInputStream("Cc: First cc <us...@james.org>\nCc: Second cc <us...@james.org>".getBytes()));
when(mailboxMessage.createFlags())
.thenReturn(new Flags());
+ when(mailboxMessage.getUid())
+ .thenReturn(MESSAGE_UID);
IndexableMessage indexableMessage = IndexableMessage.from(mailboxMessage, ImmutableList.of(new MockMailboxSession("username").getUser()),
new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.NO);
@@ -114,6 +125,8 @@ public class IndexableMessageTest {
TestId mailboxId = TestId.of(1);
when(mailboxMessage.getMailboxId())
.thenReturn(mailboxId);
+ when(mailboxMessage.getUid())
+ .thenReturn(MESSAGE_UID);
when(mailboxMessage.getFullContent())
.thenReturn(new ByteArrayInputStream("Bcc: First bcc <us...@james.org>\nBcc: Second bcc <us...@james.org>".getBytes()));
when(mailboxMessage.createFlags())
@@ -135,6 +148,8 @@ public class IndexableMessageTest {
.thenReturn(new ByteArrayInputStream("Subject: subject1\nSubject: subject2".getBytes()));
when(mailboxMessage.createFlags())
.thenReturn(new Flags());
+ when(mailboxMessage.getUid())
+ .thenReturn(MESSAGE_UID);
IndexableMessage indexableMessage = IndexableMessage.from(mailboxMessage, ImmutableList.of(new MockMailboxSession("username").getUser()),
new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.NO);
@@ -152,6 +167,8 @@ public class IndexableMessageTest {
.thenReturn(new ByteArrayInputStream("\nMy body".getBytes()));
when(mailboxMessage.createFlags())
.thenReturn(new Flags());
+ when(mailboxMessage.getUid())
+ .thenReturn(MESSAGE_UID);
IndexableMessage indexableMessage = IndexableMessage.from(mailboxMessage, ImmutableList.of(new MockMailboxSession("username").getUser()),
new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.NO);
@@ -169,6 +186,8 @@ public class IndexableMessageTest {
.thenReturn(new ByteArrayInputStream(IOUtils.toByteArray(ClassLoader.getSystemResourceAsStream("eml/mailWithHeaders.eml"))));
when(mailboxMessage.createFlags())
.thenReturn(new Flags());
+ when(mailboxMessage.getUid())
+ .thenReturn(MESSAGE_UID);
IndexableMessage indexableMessage = IndexableMessage.from(mailboxMessage, ImmutableList.of(new MockMailboxSession("username").getUser()),
new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.NO);
@@ -195,6 +214,8 @@ public class IndexableMessageTest {
.thenReturn(new ByteArrayInputStream(IOUtils.toByteArray(ClassLoader.getSystemResourceAsStream("eml/Toto.eml"))));
when(mailboxMessage.createFlags())
.thenReturn(new Flags());
+ when(mailboxMessage.getUid())
+ .thenReturn(MESSAGE_UID);
// When
IndexableMessage indexableMessage = IndexableMessage.from(mailboxMessage, ImmutableList.of(new MockMailboxSession("username").getUser()),
@@ -215,6 +236,8 @@ public class IndexableMessageTest {
.thenReturn(new ByteArrayInputStream(IOUtils.toByteArray(ClassLoader.getSystemResourceAsStream("eml/Toto.eml"))));
when(mailboxMessage.createFlags())
.thenReturn(new Flags());
+ when(mailboxMessage.getUid())
+ .thenReturn(MESSAGE_UID);
// When
IndexableMessage indexableMessage = IndexableMessage.from(mailboxMessage, ImmutableList.of(new MockMailboxSession("username").getUser()),
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java
index 596d774..6fa9d1f 100644
--- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java
+++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java
@@ -45,6 +45,7 @@ import org.apache.james.mailbox.store.extractor.DefaultTextExtractor;
import org.apache.james.mailbox.store.mail.model.MailboxMessage;
import org.apache.james.mailbox.store.mail.model.impl.PropertyBuilder;
import org.apache.james.mailbox.store.mail.model.impl.SimpleMailboxMessage;
+import org.apache.james.mailbox.store.search.MessageSearchIndex;
import org.apache.james.mailbox.tika.extractor.TikaTextExtractor;
import org.junit.Before;
import org.junit.Test;
@@ -81,7 +82,8 @@ public class MailboxMessageToElasticSearchJsonTest {
public void convertToJsonShouldThrowWhenNoUser() throws Exception {
MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson(
new DefaultTextExtractor(),
- ZoneId.of("Europe/Paris"), IndexAttachments.YES);
+ ZoneId.of("Europe/Paris"), IndexAttachments.YES,
+ MessageSearchIndex.IndexMessageId.Required);
MailboxMessage spamMail = new SimpleMailboxMessage(MESSAGE_ID,
date,
SIZE,
@@ -100,7 +102,8 @@ public class MailboxMessageToElasticSearchJsonTest {
public void spamEmailShouldBeWellConvertedToJson() throws IOException {
MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson(
new DefaultTextExtractor(),
- ZoneId.of("Europe/Paris"), IndexAttachments.YES);
+ ZoneId.of("Europe/Paris"), IndexAttachments.YES,
+ MessageSearchIndex.IndexMessageId.Required);
MailboxMessage spamMail = new SimpleMailboxMessage(MESSAGE_ID,
date,
SIZE,
@@ -120,7 +123,8 @@ public class MailboxMessageToElasticSearchJsonTest {
public void htmlEmailShouldBeWellConvertedToJson() throws IOException {
MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson(
new DefaultTextExtractor(),
- ZoneId.of("Europe/Paris"), IndexAttachments.YES);
+ ZoneId.of("Europe/Paris"), IndexAttachments.YES,
+ MessageSearchIndex.IndexMessageId.Required);
MailboxMessage htmlMail = new SimpleMailboxMessage(MESSAGE_ID,
date,
SIZE,
@@ -140,7 +144,8 @@ public class MailboxMessageToElasticSearchJsonTest {
public void pgpSignedEmailShouldBeWellConvertedToJson() throws IOException {
MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson(
new DefaultTextExtractor(),
- ZoneId.of("Europe/Paris"), IndexAttachments.YES);
+ ZoneId.of("Europe/Paris"), IndexAttachments.YES,
+ MessageSearchIndex.IndexMessageId.Required);
MailboxMessage pgpSignedMail = new SimpleMailboxMessage(MESSAGE_ID,
date,
SIZE,
@@ -160,7 +165,8 @@ public class MailboxMessageToElasticSearchJsonTest {
public void simpleEmailShouldBeWellConvertedToJson() throws IOException {
MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson(
new DefaultTextExtractor(),
- ZoneId.of("Europe/Paris"), IndexAttachments.YES);
+ ZoneId.of("Europe/Paris"), IndexAttachments.YES,
+ MessageSearchIndex.IndexMessageId.Required);
MailboxMessage mail = new SimpleMailboxMessage(MESSAGE_ID,
date,
SIZE,
@@ -181,7 +187,8 @@ public class MailboxMessageToElasticSearchJsonTest {
public void recursiveEmailShouldBeWellConvertedToJson() throws IOException {
MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson(
new DefaultTextExtractor(),
- ZoneId.of("Europe/Paris"), IndexAttachments.YES);
+ ZoneId.of("Europe/Paris"), IndexAttachments.YES,
+ MessageSearchIndex.IndexMessageId.Required);
MailboxMessage recursiveMail = new SimpleMailboxMessage(MESSAGE_ID,
date,
SIZE,
@@ -201,7 +208,8 @@ public class MailboxMessageToElasticSearchJsonTest {
public void emailWithNoInternalDateShouldUseNowDate() throws IOException {
MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson(
new DefaultTextExtractor(),
- ZoneId.of("Europe/Paris"), IndexAttachments.YES);
+ ZoneId.of("Europe/Paris"), IndexAttachments.YES,
+ MessageSearchIndex.IndexMessageId.Required);
MailboxMessage mailWithNoInternalDate = new SimpleMailboxMessage(MESSAGE_ID,
null,
SIZE,
@@ -235,7 +243,8 @@ public class MailboxMessageToElasticSearchJsonTest {
// When
MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson(
new DefaultTextExtractor(),
- ZoneId.of("Europe/Paris"), IndexAttachments.YES);
+ ZoneId.of("Europe/Paris"), IndexAttachments.YES,
+ MessageSearchIndex.IndexMessageId.Required);
String convertToJson = messageToElasticSearchJson.convertToJson(mailWithNoInternalDate, ImmutableList.of(new MockMailboxSession("username").getUser()));
// Then
@@ -262,7 +271,8 @@ public class MailboxMessageToElasticSearchJsonTest {
// When
MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson(
new DefaultTextExtractor(),
- ZoneId.of("Europe/Paris"), IndexAttachments.NO);
+ ZoneId.of("Europe/Paris"), IndexAttachments.NO,
+ MessageSearchIndex.IndexMessageId.Required);
String convertToJson = messageToElasticSearchJson.convertToJson(mailWithNoInternalDate, ImmutableList.of(new MockMailboxSession("username").getUser()));
// Then
@@ -276,7 +286,8 @@ public class MailboxMessageToElasticSearchJsonTest {
public void emailWithNoMailboxIdShouldThrow() throws IOException {
MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson(
new DefaultTextExtractor(),
- ZoneId.of("Europe/Paris"), IndexAttachments.YES);
+ ZoneId.of("Europe/Paris"), IndexAttachments.YES,
+ MessageSearchIndex.IndexMessageId.Required);
MailboxMessage mailWithNoMailboxId;
try {
mailWithNoMailboxId = new SimpleMailboxMessage(MESSAGE_ID, date,
@@ -298,7 +309,8 @@ public class MailboxMessageToElasticSearchJsonTest {
public void getUpdatedJsonMessagePartShouldBehaveWellOnEmptyFlags() throws Exception {
MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson(
new DefaultTextExtractor(),
- ZoneId.of("Europe/Paris"), IndexAttachments.YES);
+ ZoneId.of("Europe/Paris"), IndexAttachments.YES,
+ MessageSearchIndex.IndexMessageId.Required);
assertThatJson(messageToElasticSearchJson.getUpdatedJsonMessagePart(new Flags(), MOD_SEQ))
.isEqualTo("{\"modSeq\":42,\"isAnswered\":false,\"isDeleted\":false,\"isDraft\":false,\"isFlagged\":false,\"isRecent\":false,\"userFlags\":[],\"isUnread\":true}");
}
@@ -307,7 +319,8 @@ public class MailboxMessageToElasticSearchJsonTest {
public void getUpdatedJsonMessagePartShouldBehaveWellOnNonEmptyFlags() throws Exception {
MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson(
new DefaultTextExtractor(),
- ZoneId.of("Europe/Paris"), IndexAttachments.YES);
+ ZoneId.of("Europe/Paris"), IndexAttachments.YES,
+ MessageSearchIndex.IndexMessageId.Required);
assertThatJson(messageToElasticSearchJson.getUpdatedJsonMessagePart(new FlagsBuilder().add(Flags.Flag.DELETED, Flags.Flag.FLAGGED).add("user").build(), MOD_SEQ))
.isEqualTo("{\"modSeq\":42,\"isAnswered\":false,\"isDeleted\":true,\"isDraft\":false,\"isFlagged\":true,\"isRecent\":false,\"userFlags\":[\"user\"],\"isUnread\":true}");
}
@@ -316,7 +329,8 @@ public class MailboxMessageToElasticSearchJsonTest {
public void getUpdatedJsonMessagePartShouldThrowIfFlagsIsNull() throws Exception {
MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson(
new DefaultTextExtractor(),
- ZoneId.of("Europe/Paris"), IndexAttachments.YES);
+ ZoneId.of("Europe/Paris"), IndexAttachments.YES,
+ MessageSearchIndex.IndexMessageId.Required);
messageToElasticSearchJson.getUpdatedJsonMessagePart(null, MOD_SEQ);
}
@@ -324,7 +338,8 @@ public class MailboxMessageToElasticSearchJsonTest {
public void spamEmailShouldBeWellConvertedToJsonWithApacheTika() throws IOException {
MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson(
new TikaTextExtractor(),
- ZoneId.of("Europe/Paris"), IndexAttachments.YES);
+ ZoneId.of("Europe/Paris"), IndexAttachments.YES,
+ MessageSearchIndex.IndexMessageId.Required);
MailboxMessage spamMail = new SimpleMailboxMessage(MESSAGE_ID, date,
SIZE,
BODY_START_OCTET,
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
----------------------------------------------------------------------
diff --git a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
index 7dee3cf..9b3c0c1 100644
--- a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
+++ b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
@@ -40,13 +40,13 @@ import javax.inject.Inject;
import javax.mail.Flags;
import javax.mail.Flags.Flag;
+import org.apache.james.mailbox.MailboxManager;
import org.apache.james.mailbox.MailboxManager.SearchCapabilities;
import org.apache.james.mailbox.MailboxSession;
import org.apache.james.mailbox.MessageUid;
import org.apache.james.mailbox.exception.MailboxException;
import org.apache.james.mailbox.exception.UnsupportedSearchException;
import org.apache.james.mailbox.model.MailboxId;
-import org.apache.james.mailbox.model.MailboxId.Factory;
import org.apache.james.mailbox.model.MessageId;
import org.apache.james.mailbox.model.MessageRange;
import org.apache.james.mailbox.model.MultimailboxesSearchQuery;
@@ -68,6 +68,7 @@ import org.apache.james.mailbox.store.mail.MessageMapperFactory;
import org.apache.james.mailbox.store.mail.model.Mailbox;
import org.apache.james.mailbox.store.mail.model.MailboxMessage;
import org.apache.james.mailbox.store.search.ListeningMessageSearchIndex;
+import org.apache.james.mailbox.store.search.MessageSearchIndex;
import org.apache.james.mailbox.store.search.SearchUtil;
import org.apache.james.mime4j.MimeException;
import org.apache.james.mime4j.dom.Header;
@@ -118,6 +119,7 @@ import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
import com.google.common.base.Function;
+import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
@@ -346,33 +348,42 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
private final static SortField FIRST_FROM_MAILBOX_DISPLAY_SORT = new SortField(FIRST_FROM_MAILBOX_DISPLAY_FIELD, SortField.STRING);
private final static SortField FIRST_FROM_MAILBOX_DISPLAY_SORT_REVERSE = new SortField(FIRST_FROM_MAILBOX_DISPLAY_FIELD, SortField.STRING, true);
- private final Factory mailboxIdFactory;
+ private final MailboxId.Factory mailboxIdFactory;
private final MessageId.Factory messageIdFactory;
private final IndexWriter writer;
+ private final IndexMessageId shouldIndexMessageId;
private int maxQueryResults = DEFAULT_MAX_QUERY_RESULTS;
private boolean suffixMatch = false;
@Inject
- public LuceneMessageSearchIndex(MessageMapperFactory factory, Factory mailboxIdFactory, Directory directory, MessageId.Factory messageIdFactory) throws CorruptIndexException, LockObtainFailedException, IOException {
- this(factory, mailboxIdFactory, directory, false, true, messageIdFactory);
+ public LuceneMessageSearchIndex(MessageMapperFactory factory, MailboxId.Factory mailboxIdFactory, Directory directory, MessageId.Factory messageIdFactory, MailboxManager mailboxManager) throws CorruptIndexException, LockObtainFailedException, IOException {
+ this(factory, mailboxIdFactory, directory, false, true, messageIdFactory, indexMessageId(mailboxManager));
+ }
+
+ private static MessageSearchIndex.IndexMessageId indexMessageId(MailboxManager mailboxManager) {
+ if (mailboxManager.getSupportedMessageCapabilities().contains(MailboxManager.MessageCapabilities.UniqueID)) {
+ return MessageSearchIndex.IndexMessageId.Required;
+ }
+ return MessageSearchIndex.IndexMessageId.Optional;
}
-
- public LuceneMessageSearchIndex(MessageMapperFactory factory, Factory mailboxIdFactory, Directory directory, boolean dropIndexOnStart, boolean lenient, MessageId.Factory messageIdFactory) throws CorruptIndexException, LockObtainFailedException, IOException {
+ public LuceneMessageSearchIndex(MessageMapperFactory factory, MailboxId.Factory mailboxIdFactory, Directory directory, boolean dropIndexOnStart, boolean lenient, MessageId.Factory messageIdFactory, IndexMessageId shouldIndexMessageId) throws CorruptIndexException, LockObtainFailedException, IOException {
super(factory);
this.mailboxIdFactory = mailboxIdFactory;
this.messageIdFactory = messageIdFactory;
this.writer = new IndexWriter(directory, createConfig(createAnalyzer(lenient), dropIndexOnStart));
+ this.shouldIndexMessageId = shouldIndexMessageId;
}
- public LuceneMessageSearchIndex(MessageMapperFactory factory, Factory mailboxIdFactory, MessageId.Factory messageIdFactory, IndexWriter writer) {
+ public LuceneMessageSearchIndex(MessageMapperFactory factory, MailboxId.Factory mailboxIdFactory, MessageId.Factory messageIdFactory, IndexWriter writer, IndexMessageId shouldIndexMessageId) {
super(factory);
this.mailboxIdFactory = mailboxIdFactory;
this.messageIdFactory = messageIdFactory;
this.writer = writer;
+ this.shouldIndexMessageId = shouldIndexMessageId;
}
@Override
@@ -461,7 +472,7 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
.transform(new Function<SearchResult, MessageId>() {
@Override
public MessageId apply(SearchResult input) {
- return input.getMessageId();
+ return input.getMessageId().get();
}
})
.limit(Long.valueOf(limit).intValue())
@@ -492,7 +503,7 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
Document doc = searcher.doc(sDoc.doc);
MessageUid uid = MessageUid.of(Long.valueOf(doc.get(UID_FIELD)));
MailboxId mailboxId = mailboxIdFactory.fromString(doc.get(MAILBOX_ID_FIELD));
- MessageId messageId = messageIdFactory.fromString(doc.get(MESSAGE_ID_FIELD));
+ Optional<MessageId> messageId = toMessageId(Optional.fromNullable(doc.get(MESSAGE_ID_FIELD)));
results.add(new SearchResult(messageId, mailboxId, uid));
}
} catch (IOException e) {
@@ -508,7 +519,14 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
}
return results.build();
}
-
+
+ private Optional<MessageId> toMessageId(Optional<String> messageIdField) {
+ if (messageIdField.isPresent()) {
+ return Optional.of(messageIdFactory.fromString(messageIdField.get()));
+ }
+ return Optional.absent();
+ }
+
private Query buildQueryFromMailboxes(ImmutableSet<MailboxId> mailboxIds) {
if (mailboxIds.isEmpty()) {
return new MatchAllDocsQuery();
@@ -535,8 +553,11 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
// TODO: Better handling
doc.add(new Field(MAILBOX_ID_FIELD, membership.getMailboxId().serialize().toUpperCase(Locale.ENGLISH), Store.YES, Index.NOT_ANALYZED));
doc.add(new NumericField(UID_FIELD,Store.YES, true).setLongValue(membership.getUid().asLong()));
- doc.add(new Field(MESSAGE_ID_FIELD, membership.getMessageId().serialize(), Store.YES, Index.NOT_ANALYZED));
-
+
+ if (shouldIndexMessageId == IndexMessageId.Required) {
+ doc.add(new Field(MESSAGE_ID_FIELD, membership.getMessageId().serialize(), Store.YES, Index.NOT_ANALYZED));
+ }
+
// create an unqiue key for the document which can be used later on updates to find the document
doc.add(new Field(ID_FIELD, membership.getMailboxId().serialize().toUpperCase(Locale.ENGLISH) +"-" + Long.toString(membership.getUid().asLong()), Store.YES, Index.NOT_ANALYZED));
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/lucene/src/main/resources/META-INF/spring/mailbox-index-lucene.xml
----------------------------------------------------------------------
diff --git a/mailbox/lucene/src/main/resources/META-INF/spring/mailbox-index-lucene.xml b/mailbox/lucene/src/main/resources/META-INF/spring/mailbox-index-lucene.xml
index 1ed21f4..e551478 100644
--- a/mailbox/lucene/src/main/resources/META-INF/spring/mailbox-index-lucene.xml
+++ b/mailbox/lucene/src/main/resources/META-INF/spring/mailbox-index-lucene.xml
@@ -35,6 +35,7 @@
<constructor-arg index="1" ref="mailboxIdFactory"/>
<constructor-arg index="2" type="org.apache.lucene.store.Directory" ref="fsDirectory"/>
<constructor-arg index="3" ref="messageIdFactory"/>
+ <constructor-arg index="4" ref="mailboxmanager"/>
<property name="enableSuffixMatch" value="true"/>
</bean>
<bean id="fsDirectory" class="org.apache.lucene.store.FSDirectory" factory-method="open">
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java
----------------------------------------------------------------------
diff --git a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java
index 82d8cba..054ec1c 100644
--- a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java
+++ b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java
@@ -50,6 +50,7 @@ import org.apache.james.mailbox.model.TestMessageId;
import org.apache.james.mailbox.store.MessageBuilder;
import org.apache.james.mailbox.store.SimpleMailboxMembership;
import org.apache.james.mailbox.store.mail.model.Mailbox;
+import org.apache.james.mailbox.store.search.MessageSearchIndex;
import org.apache.lucene.store.RAMDirectory;
import org.junit.Before;
import org.junit.Test;
@@ -99,7 +100,7 @@ public class LuceneMailboxMessageSearchIndexTest {
id3 = factory.generate();
id4 = factory.generate();
id5 = factory.generate();
- index = new LuceneMessageSearchIndex(null, new TestId.Factory(), new RAMDirectory(), true, useLenient(), factory);
+ index = new LuceneMessageSearchIndex(null, new TestId.Factory(), new RAMDirectory(), true, useLenient(), factory, MessageSearchIndex.IndexMessageId.Required);
index.setEnableSuffixMatch(true);
Map<String, String> headersSubject = new HashMap<String, String>();
headersSubject.put("Subject", "test (fwd)");
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndexTest.java
----------------------------------------------------------------------
diff --git a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndexTest.java b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndexTest.java
index 864a7a0..dfdaffd 100644
--- a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndexTest.java
+++ b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndexTest.java
@@ -45,7 +45,6 @@ public class LuceneMessageSearchIndexTest extends AbstractMessageSearchIndexTest
protected void initializeMailboxManager() throws Exception {
TestMessageId.Factory messageIdFactory = new TestMessageId.Factory();
MailboxSessionMapperFactory mapperFactory = new InMemoryMailboxSessionMapperFactory();
- messageSearchIndex = new LuceneMessageSearchIndex(mapperFactory, new InMemoryId.Factory(), new RAMDirectory(), messageIdFactory);
storeMailboxManager = new InMemoryMailboxManager(
mapperFactory,
new FakeAuthenticator(),
@@ -54,6 +53,7 @@ public class LuceneMessageSearchIndexTest extends AbstractMessageSearchIndexTest
new SimpleGroupMembershipResolver(),
new MessageParser(),
messageIdFactory);
+ messageSearchIndex = new LuceneMessageSearchIndex(mapperFactory, new InMemoryId.Factory(), new RAMDirectory(), messageIdFactory, storeMailboxManager);
storeMailboxManager.setMessageSearchIndex(messageSearchIndex);
storeMailboxManager.init();
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/spring/src/test/resources/META-INF/spring/mailbox-index-lucene.xml
----------------------------------------------------------------------
diff --git a/mailbox/spring/src/test/resources/META-INF/spring/mailbox-index-lucene.xml b/mailbox/spring/src/test/resources/META-INF/spring/mailbox-index-lucene.xml
index cf51b98..31c5790 100644
--- a/mailbox/spring/src/test/resources/META-INF/spring/mailbox-index-lucene.xml
+++ b/mailbox/spring/src/test/resources/META-INF/spring/mailbox-index-lucene.xml
@@ -35,10 +35,12 @@
<constructor-arg index="1" ref="mailboxIdFactory"/>
<constructor-arg index="2" type="org.apache.lucene.store.Directory" ref="fsDirectory"/>
<constructor-arg index="3" ref="messageIdFactory"/>
+ <constructor-arg index="4" ref="mailboxmanager"/>
<property name="enableSuffixMatch" value="true"/>
</bean>
<bean id="fsDirectory" class="org.apache.lucene.store.FSDirectory" factory-method="open">
<constructor-arg index="0" value="../var/store/lucene"/>
</bean>
<alias name="jpa-mailboxIdFactory" alias="mailboxIdFactory"/>
+ <alias name="jpa-mailboxmanager" alias="mailboxmanager"/>
</beans>
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearchIndex.java
----------------------------------------------------------------------
diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearchIndex.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearchIndex.java
index 3df16bd..162598a 100644
--- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearchIndex.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearchIndex.java
@@ -33,6 +33,8 @@ import org.apache.james.mailbox.model.MultimailboxesSearchQuery;
import org.apache.james.mailbox.model.SearchQuery;
import org.apache.james.mailbox.store.mail.model.Mailbox;
+import com.google.common.base.Optional;
+
/**
* An index which can be used to search for MailboxMessage UID's that match a {@link SearchQuery}.
*
@@ -41,6 +43,11 @@ import org.apache.james.mailbox.store.mail.model.Mailbox;
*/
public interface MessageSearchIndex {
+ enum IndexMessageId {
+ Required,
+ Optional
+ }
+
/**
* Return all uids of the previous indexed {@link Mailbox}'s which match the {@link SearchQuery}
*/
@@ -54,17 +61,17 @@ public interface MessageSearchIndex {
EnumSet<MailboxManager.SearchCapabilities> getSupportedCapabilities();
class SearchResult {
- private final MessageId messageId;
+ private final Optional<MessageId> messageId;
private final MailboxId mailboxId;
private final MessageUid messageUid;
- public SearchResult(MessageId messageId, MailboxId mailboxId, MessageUid messageUid) {
+ public SearchResult(Optional<MessageId> messageId, MailboxId mailboxId, MessageUid messageUid) {
this.messageId = messageId;
this.mailboxId = mailboxId;
this.messageUid = messageUid;
}
- public MessageId getMessageId() {
+ public Optional<MessageId> getMessageId() {
return messageId;
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearches.java
----------------------------------------------------------------------
diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearches.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearches.java
index 2f69adb..6fe58b5 100644
--- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearches.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearches.java
@@ -68,6 +68,7 @@ import org.apache.james.mime4j.message.HeaderImpl;
import org.apache.james.mime4j.utils.search.MessageMatcher;
import com.google.common.base.Function;
+import com.google.common.base.Optional;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -114,7 +115,7 @@ public class MessageSearches implements Iterable<SimpleMessageSearchIndex.Search
@Override
public SimpleMessageSearchIndex.SearchResult apply(MailboxMessage input) {
return new SimpleMessageSearchIndex.SearchResult(
- input.getMessageId(),
+ Optional.of(input.getMessageId()),
input.getMailboxId(),
input.getUid());
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndex.java
----------------------------------------------------------------------
diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndex.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndex.java
index 537164e..3d3ee63 100644
--- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndex.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndex.java
@@ -182,7 +182,7 @@ public class SimpleMessageSearchIndex implements MessageSearchIndex {
return new Function<SearchResult, MessageId>() {
@Override
public MessageId apply(SearchResult input) {
- return input.getMessageId();
+ return input.getMessageId().get();
}
};
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/test/resources/eml/htmlMail.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/htmlMail.json b/mailbox/store/src/test/resources/eml/htmlMail.json
index fdc1465..1655f5e 100644
--- a/mailbox/store/src/test/resources/eml/htmlMail.json
+++ b/mailbox/store/src/test/resources/eml/htmlMail.json
@@ -1,5 +1,5 @@
{
- "id":"184",
+ "messageId":"184",
"uid":25,
"mailboxId":"18",
"modSeq":42,
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/test/resources/eml/mail.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/mail.json b/mailbox/store/src/test/resources/eml/mail.json
index c734e45..76068a5 100644
--- a/mailbox/store/src/test/resources/eml/mail.json
+++ b/mailbox/store/src/test/resources/eml/mail.json
@@ -1,5 +1,5 @@
{
- "id":"184",
+ "messageId":"184",
"uid":25,
"mailboxId": "18",
"modSeq": 42,
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/test/resources/eml/nonTextual.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/nonTextual.json b/mailbox/store/src/test/resources/eml/nonTextual.json
index f15f412..9d1aa9c 100644
--- a/mailbox/store/src/test/resources/eml/nonTextual.json
+++ b/mailbox/store/src/test/resources/eml/nonTextual.json
@@ -1,5 +1,5 @@
{
- "id":"184",
+ "messageId":"184",
"uid":25,
"mailboxId":"18",
"modSeq":42,
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/test/resources/eml/pgpSignedMail.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/pgpSignedMail.json b/mailbox/store/src/test/resources/eml/pgpSignedMail.json
index 91bdc42..0b00109 100644
--- a/mailbox/store/src/test/resources/eml/pgpSignedMail.json
+++ b/mailbox/store/src/test/resources/eml/pgpSignedMail.json
@@ -1,5 +1,5 @@
{
- "id":"184",
+ "messageId":"184",
"uid":25,
"mailboxId": "18",
"modSeq": 42,
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/test/resources/eml/recursiveMail.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/recursiveMail.json b/mailbox/store/src/test/resources/eml/recursiveMail.json
index 4ec9942..f0ae82e 100644
--- a/mailbox/store/src/test/resources/eml/recursiveMail.json
+++ b/mailbox/store/src/test/resources/eml/recursiveMail.json
@@ -1,5 +1,5 @@
{
- "id":"184",
+ "messageId":"184",
"uid":25,
"mailboxId": "18",
"modSeq": 42,
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json b/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json
index 06935fd..9dfb964 100644
--- a/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json
+++ b/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json
@@ -1,5 +1,5 @@
{
- "id":"184",
+ "messageId":"184",
"uid":25,
"mailboxId": "18",
"modSeq": 42,
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/test/resources/eml/spamMail.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/spamMail.json b/mailbox/store/src/test/resources/eml/spamMail.json
index d8ede1e..b71a5ce 100644
--- a/mailbox/store/src/test/resources/eml/spamMail.json
+++ b/mailbox/store/src/test/resources/eml/spamMail.json
@@ -1,5 +1,5 @@
{
- "id":"184",
+ "messageId":"184",
"uid":25,
"mailboxId": "18",
"modSeq": 42,
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java
----------------------------------------------------------------------
diff --git a/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java b/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java
index 4a50a02..e6a09a7 100644
--- a/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java
+++ b/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java
@@ -21,6 +21,7 @@ package org.apache.james.mpt.imapmailbox.elasticsearch.host;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.time.ZoneId;
import java.util.concurrent.Executors;
import org.apache.commons.io.FileUtils;
@@ -57,6 +58,7 @@ import org.apache.james.mailbox.store.extractor.DefaultTextExtractor;
import org.apache.james.mailbox.store.mail.model.impl.MessageParser;
import org.apache.james.mailbox.store.quota.DefaultQuotaRootResolver;
import org.apache.james.mailbox.store.quota.NoQuotaManager;
+import org.apache.james.mailbox.store.search.MessageSearchIndex;
import org.apache.james.mpt.api.ImapFeatures;
import org.apache.james.mpt.api.ImapFeatures.Feature;
import org.apache.james.mpt.host.JamesImapHostSystem;
@@ -111,7 +113,7 @@ public class ElasticSearchHostSystem extends JamesImapHostSystem {
factory,
new ElasticSearchIndexer(client, new DeleteByQueryPerformer(client, Executors.newSingleThreadExecutor())),
new ElasticSearchSearcher(client, new QueryConverter(new CriterionConverter()), new InMemoryId.Factory(), messageIdFactory),
- new MessageToElasticSearchJson(new DefaultTextExtractor(), IndexAttachments.YES));
+ new MessageToElasticSearchJson(new DefaultTextExtractor(), ZoneId.systemDefault(), IndexAttachments.YES, MessageSearchIndex.IndexMessageId.Required));
MailboxACLResolver aclResolver = new UnionMailboxACLResolver();
GroupMembershipResolver groupMembershipResolver = new SimpleGroupMembershipResolver();
http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mpt/impl/imap-mailbox/lucenesearch/src/test/java/org/apache/james/mpt/imapmailbox/lucenesearch/host/LuceneSearchHostSystem.java
----------------------------------------------------------------------
diff --git a/mpt/impl/imap-mailbox/lucenesearch/src/test/java/org/apache/james/mpt/imapmailbox/lucenesearch/host/LuceneSearchHostSystem.java b/mpt/impl/imap-mailbox/lucenesearch/src/test/java/org/apache/james/mpt/imapmailbox/lucenesearch/host/LuceneSearchHostSystem.java
index 63bd19b..40b6e33 100644
--- a/mpt/impl/imap-mailbox/lucenesearch/src/test/java/org/apache/james/mpt/imapmailbox/lucenesearch/host/LuceneSearchHostSystem.java
+++ b/mpt/impl/imap-mailbox/lucenesearch/src/test/java/org/apache/james/mpt/imapmailbox/lucenesearch/host/LuceneSearchHostSystem.java
@@ -151,15 +151,16 @@ public class LuceneSearchHostSystem extends JamesImapHostSystem {
try {
JPAId.Factory mailboxIdFactory = new Factory();
FSDirectory fsDirectory = FSDirectory.open(tempFile);
- LuceneMessageSearchIndex searchIndex = new LuceneMessageSearchIndex(factory, mailboxIdFactory, fsDirectory, messageIdFactory);
- searchIndex.setEnableSuffixMatch(true);
+ MessageId.Factory messageIdFactory = new DefaultMessageId.Factory();
MailboxACLResolver aclResolver = new UnionMailboxACLResolver();
GroupMembershipResolver groupMembershipResolver = new SimpleGroupMembershipResolver();
MessageParser messageParser = new MessageParser();
- MessageId.Factory messageIdFactory = new DefaultMessageId.Factory();
mailboxManager = new OpenJPAMailboxManager(factory, userManager, locker, false, aclResolver, groupMembershipResolver, messageParser, messageIdFactory);
+
+ LuceneMessageSearchIndex searchIndex = new LuceneMessageSearchIndex(factory, mailboxIdFactory, fsDirectory, messageIdFactory, mailboxManager);
+ searchIndex.setEnableSuffixMatch(true);
mailboxManager.setMessageSearchIndex(searchIndex);
mailboxManager.init();
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org