You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@james.apache.org by rc...@apache.org on 2020/10/02 08:48:43 UTC
[james-project] 07/08: JAMES-3377 Remove unused text criterion
This is an automated email from the ASF dual-hosted git repository.
rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 6830a1aeb997d3ec4dfd2acbbcfe1c8c200433fe
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Fri Oct 2 05:00:06 2020 +0200
JAMES-3377 Remove unused text criterion
We uses a "or" on the JMAP side, both on draft and on RFC-8621
This unlock an indexed message size reduction up to 25% (the actual improvement is bigger as this is an indexed field!)
---
.../apache/james/mailbox/model/SearchQuery.java | 21 --
.../elasticsearch/MailboxMappingFactory.java | 14 --
.../elasticsearch/json/IndexableMessage.java | 10 +-
.../elasticsearch/query/CriterionConverter.java | 5 -
.../elasticsearch/json/IndexableMessageTest.java | 264 ---------------------
.../store/search/SimpleMessageSearchIndexTest.java | 20 --
.../mailbox/store/search/MessageSearches.java | 8 -
.../search/AbstractMessageSearchIndexTest.java | 141 -----------
...ithNonIndexableAttachmentWithoutAttachment.json | 1 -
mailbox/store/src/test/resources/eml/htmlMail.json | 3 +-
.../src/test/resources/eml/invalidCharset.json | 1 -
mailbox/store/src/test/resources/eml/mail.json | 3 +-
.../store/src/test/resources/eml/nonTextual.json | 1 -
.../src/test/resources/eml/pgpSignedMail.json | 3 +-
.../src/test/resources/eml/recursiveMail.json | 3 +-
.../eml/recursiveMailWithoutAttachments.json | 3 +-
mailbox/store/src/test/resources/eml/spamMail.json | 3 +-
17 files changed, 7 insertions(+), 497 deletions(-)
diff --git a/mailbox/api/src/main/java/org/apache/james/mailbox/model/SearchQuery.java b/mailbox/api/src/main/java/org/apache/james/mailbox/model/SearchQuery.java
index d761753..c82f2d6 100644
--- a/mailbox/api/src/main/java/org/apache/james/mailbox/model/SearchQuery.java
+++ b/mailbox/api/src/main/java/org/apache/james/mailbox/model/SearchQuery.java
@@ -504,22 +504,6 @@ public class SearchQuery implements Serializable {
}
/**
- * Creates a filter matching messages which contains the given text either
- * within the headers (From, To, Cc, Bcc & Subject) and text / html bodies.
- * Implementations may choose to ignore mime parts which cannot be decoded to text.
- *
- * All to-compared Strings MUST BE converted to uppercase before doing so
- * (this also include the search value)
- *
- * @param value
- * search value
- * @return <code>Criterion</code>, not null
- */
- public static Criterion textContains(String value) {
- return new TextCriterion(value, Scope.TEXT);
- }
-
- /**
* Creates a filter matching messages which contains the given text within
* the body. Implementations may choose to ignore mime parts which cannot be
* decoded to text.
@@ -1050,11 +1034,6 @@ public class SearchQuery implements Serializable {
/** Only message body content */
BODY,
- /** Headers: From, To, Cc, Bcc & Subjects
- * plus text/plain & text/html part
- */
- TEXT,
-
/** Full message content including headers and attachments */
FULL,
/** Attachment content */
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxMappingFactory.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxMappingFactory.java
index 22d915f..aeb6782 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxMappingFactory.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxMappingFactory.java
@@ -21,7 +21,6 @@ package org.apache.james.mailbox.elasticsearch;
import static org.apache.james.backends.es.IndexCreationFactory.CASE_INSENSITIVE;
import static org.apache.james.backends.es.IndexCreationFactory.KEEP_MAIL_AND_URL;
-import static org.apache.james.backends.es.IndexCreationFactory.SNOWBALL_KEEP_MAIL_AND_URL;
import static org.apache.james.backends.es.NodeMappingFactory.ANALYZER;
import static org.apache.james.backends.es.NodeMappingFactory.BOOLEAN;
import static org.apache.james.backends.es.NodeMappingFactory.FIELDS;
@@ -35,7 +34,6 @@ import static org.apache.james.backends.es.NodeMappingFactory.RAW;
import static org.apache.james.backends.es.NodeMappingFactory.REQUIRED;
import static org.apache.james.backends.es.NodeMappingFactory.ROUTING;
import static org.apache.james.backends.es.NodeMappingFactory.SEARCH_ANALYZER;
-import static org.apache.james.backends.es.NodeMappingFactory.SNOWBALL;
import static org.apache.james.backends.es.NodeMappingFactory.SPLIT_EMAIL;
import static org.apache.james.backends.es.NodeMappingFactory.TYPE;
import static org.apache.james.mailbox.elasticsearch.json.JsonMessageConstants.ATTACHMENTS;
@@ -329,18 +327,6 @@ public class MailboxMappingFactory {
.field(TYPE, BOOLEAN)
.endObject()
- .startObject(TEXT)
- .field(TYPE, TEXT)
- .field(ANALYZER, SNOWBALL_KEEP_MAIL_AND_URL)
- .startObject(FIELDS)
- .startObject(SPLIT_EMAIL)
- .field(TYPE, TEXT)
- .field(ANALYZER, SNOWBALL)
- .field(SEARCH_ANALYZER, SNOWBALL_KEEP_MAIL_AND_URL)
- .endObject()
- .endObject()
- .endObject()
-
.startObject(ATTACHMENTS)
.startObject(PROPERTIES)
.startObject(Attachment.FILENAME)
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java
index e24f75a..17941bc 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java
@@ -171,7 +171,6 @@ public class IndexableMessage {
size,
subjects,
subType,
- text,
to,
uid,
userFlags,
@@ -215,7 +214,6 @@ public class IndexableMessage {
private final long size;
private final Subjects subjects;
private final String subType;
- private final String text;
private final EMailers to;
private final long uid;
private final String[] userFlags;
@@ -242,7 +240,7 @@ public class IndexableMessage {
String sentDate,
long size,
Subjects subjects,
- String subType, String text,
+ String subType,
EMailers to,
long uid,
String[] userFlags,
@@ -270,7 +268,6 @@ public class IndexableMessage {
this.size = size;
this.subjects = subjects;
this.subType = subType;
- this.text = text;
this.to = to;
this.uid = uid;
this.userFlags = userFlags;
@@ -362,11 +359,6 @@ public class IndexableMessage {
return subType;
}
- @JsonProperty(JsonMessageConstants.TEXT)
- public String getText() {
- return text;
- }
-
@JsonProperty(JsonMessageConstants.TO)
public EMailers getTo() {
return to;
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/CriterionConverter.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/CriterionConverter.java
index e654df4..8937807 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/CriterionConverter.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/CriterionConverter.java
@@ -155,11 +155,6 @@ public class CriterionConverter {
.should(matchQuery(JsonMessageConstants.HTML_BODY + "." + SPLIT_EMAIL,
textCriterion.getOperator().getValue()))
.should(matchQuery(JsonMessageConstants.HTML_BODY, textCriterion.getOperator().getValue()));
- case TEXT:
- return boolQuery()
- .should(matchQuery(JsonMessageConstants.TEXT, textCriterion.getOperator().getValue()))
- .should(matchQuery(JsonMessageConstants.TEXT + "." + SPLIT_EMAIL,
- textCriterion.getOperator().getValue()));
case FULL:
return boolQuery()
.should(matchQuery(JsonMessageConstants.TEXT_BODY, textCriterion.getOperator().getValue()))
diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java
index 801ac5f..3c97a8e 100644
--- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java
+++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java
@@ -24,7 +24,6 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.time.ZoneId;
import java.util.Optional;
@@ -49,7 +48,6 @@ import org.apache.james.mailbox.tika.TikaExtension;
import org.apache.james.mailbox.tika.TikaHttpClientImpl;
import org.apache.james.mailbox.tika.TikaTextExtractor;
import org.apache.james.metrics.tests.RecordingMetricFactory;
-import org.assertj.core.api.iterable.Extractor;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
@@ -75,238 +73,6 @@ class IndexableMessageTest {
}
@Test
- void textShouldBeEmptyWhenNoMatchingHeaders() throws Exception {
- MailboxMessage mailboxMessage = mock(MailboxMessage.class);
- TestId mailboxId = TestId.of(1);
- when(mailboxMessage.getMailboxId())
- .thenReturn(mailboxId);
- when(mailboxMessage.getMessageId())
- .thenReturn(InMemoryMessageId.of(42));
- when(mailboxMessage.getModSeq())
- .thenReturn(ModSeq.first());
- when(mailboxMessage.getFullContent())
- .thenReturn(new ByteArrayInputStream("".getBytes()));
- when(mailboxMessage.createFlags())
- .thenReturn(new Flags());
- when(mailboxMessage.getUid())
- .thenReturn(MESSAGE_UID);
-
- IndexableMessage indexableMessage = IndexableMessage.builder()
- .message(mailboxMessage)
- .extractor(new DefaultTextExtractor())
- .zoneId(ZoneId.of("Europe/Paris"))
- .indexAttachments(IndexAttachments.NO)
- .build();
-
- assertThat(indexableMessage.getText()).isEmpty();
- }
-
- @Test
- void textShouldContainsFromWhenFrom() throws Exception {
- MailboxMessage mailboxMessage = mock(MailboxMessage.class);
- TestId mailboxId = TestId.of(1);
- when(mailboxMessage.getMailboxId())
- .thenReturn(mailboxId);
- when(mailboxMessage.getModSeq())
- .thenReturn(ModSeq.first());
- when(mailboxMessage.getMessageId())
- .thenReturn(InMemoryMessageId.of(42));
- when(mailboxMessage.getFullContent())
- .thenReturn(new ByteArrayInputStream("From: First user <us...@james.org>\nFrom: Second user <us...@james.org>".getBytes()));
- when(mailboxMessage.createFlags())
- .thenReturn(new Flags());
- when(mailboxMessage.getUid())
- .thenReturn(MESSAGE_UID);
-
- IndexableMessage indexableMessage = IndexableMessage.builder()
- .message(mailboxMessage)
- .extractor(new DefaultTextExtractor())
- .zoneId(ZoneId.of("Europe/Paris"))
- .indexAttachments(IndexAttachments.NO)
- .build();
-
- assertThat(indexableMessage.getText())
- .contains("Second user user2@james.org")
- .contains("First user user@james.org");
- }
-
- @Test
- void textShouldContainsToWhenTo() throws Exception {
- MailboxMessage mailboxMessage = mock(MailboxMessage.class);
- TestId mailboxId = TestId.of(1);
- when(mailboxMessage.getMailboxId())
- .thenReturn(mailboxId);
- when(mailboxMessage.getModSeq())
- .thenReturn(ModSeq.first());
- when(mailboxMessage.getMessageId())
- .thenReturn(InMemoryMessageId.of(42));
- when(mailboxMessage.getFullContent())
- .thenReturn(new ByteArrayInputStream("To: First to <us...@james.org>\nTo: Second to <us...@james.org>".getBytes()));
- when(mailboxMessage.createFlags())
- .thenReturn(new Flags());
- when(mailboxMessage.getUid())
- .thenReturn(MESSAGE_UID);
-
- IndexableMessage indexableMessage = IndexableMessage.builder()
- .message(mailboxMessage)
- .extractor(new DefaultTextExtractor())
- .zoneId(ZoneId.of("Europe/Paris"))
- .indexAttachments(IndexAttachments.NO)
- .build();
-
- assertThat(indexableMessage.getText())
- .contains("First to user@james.org")
- .contains("Second to user2@james.org");
- }
-
- @Test
- void textShouldContainsCcWhenCc() throws Exception {
- MailboxMessage mailboxMessage = mock(MailboxMessage.class);
- TestId mailboxId = TestId.of(1);
- when(mailboxMessage.getMailboxId())
- .thenReturn(mailboxId);
- when(mailboxMessage.getMessageId())
- .thenReturn(InMemoryMessageId.of(42));
- when(mailboxMessage.getFullContent())
- .thenReturn(new ByteArrayInputStream("Cc: First cc <us...@james.org>\nCc: Second cc <us...@james.org>".getBytes()));
- when(mailboxMessage.createFlags())
- .thenReturn(new Flags());
- when(mailboxMessage.getUid())
- .thenReturn(MESSAGE_UID);
- when(mailboxMessage.getModSeq())
- .thenReturn(ModSeq.first());
-
- IndexableMessage indexableMessage = IndexableMessage.builder()
- .message(mailboxMessage)
- .extractor(new DefaultTextExtractor())
- .zoneId(ZoneId.of("Europe/Paris"))
- .indexAttachments(IndexAttachments.NO)
- .build();
-
- assertThat(indexableMessage.getText())
- .contains("Second cc user2@james.org")
- .contains("First cc user@james.org");
- }
-
- @Test
- void textShouldContainsBccWhenBcc() throws Exception {
- MailboxMessage mailboxMessage = mock(MailboxMessage.class);
- TestId mailboxId = TestId.of(1);
- when(mailboxMessage.getMailboxId())
- .thenReturn(mailboxId);
- when(mailboxMessage.getMessageId())
- .thenReturn(InMemoryMessageId.of(42));
- when(mailboxMessage.getUid())
- .thenReturn(MESSAGE_UID);
- when(mailboxMessage.getFullContent())
- .thenReturn(new ByteArrayInputStream("Bcc: First bcc <us...@james.org>\nBcc: Second bcc <us...@james.org>".getBytes()));
- when(mailboxMessage.createFlags())
- .thenReturn(new Flags());
- when(mailboxMessage.getModSeq())
- .thenReturn(ModSeq.first());
-
- IndexableMessage indexableMessage = IndexableMessage.builder()
- .message(mailboxMessage)
- .extractor(new DefaultTextExtractor())
- .zoneId(ZoneId.of("Europe/Paris"))
- .indexAttachments(IndexAttachments.NO)
- .build();
-
- assertThat(indexableMessage.getText())
- .contains("Second bcc user2@james.org")
- .contains("First bcc user@james.org");
- }
-
- @Test
- void textShouldContainsSubjectsWhenSubjects() throws Exception {
- MailboxMessage mailboxMessage = mock(MailboxMessage.class);
- TestId mailboxId = TestId.of(1);
- when(mailboxMessage.getMailboxId())
- .thenReturn(mailboxId);
- when(mailboxMessage.getModSeq())
- .thenReturn(ModSeq.first());
- when(mailboxMessage.getMessageId())
- .thenReturn(InMemoryMessageId.of(42));
- when(mailboxMessage.getFullContent())
- .thenReturn(new ByteArrayInputStream("Subject: subject1\nSubject: subject2".getBytes()));
- when(mailboxMessage.createFlags())
- .thenReturn(new Flags());
- when(mailboxMessage.getUid())
- .thenReturn(MESSAGE_UID);
-
- IndexableMessage indexableMessage = IndexableMessage.builder()
- .message(mailboxMessage)
- .extractor(new DefaultTextExtractor())
- .zoneId(ZoneId.of("Europe/Paris"))
- .indexAttachments(IndexAttachments.NO)
- .build();
-
- assertThat(indexableMessage.getText()).isEqualTo("subject1 subject2");
- }
-
- @Test
- void textShouldContainsBodyWhenBody() throws Exception {
- MailboxMessage mailboxMessage = mock(MailboxMessage.class);
- TestId mailboxId = TestId.of(1);
- when(mailboxMessage.getMailboxId())
- .thenReturn(mailboxId);
- when(mailboxMessage.getModSeq())
- .thenReturn(ModSeq.first());
- when(mailboxMessage.getMessageId())
- .thenReturn(InMemoryMessageId.of(42));
- when(mailboxMessage.getFullContent())
- .thenReturn(new ByteArrayInputStream("\nMy body".getBytes()));
- when(mailboxMessage.createFlags())
- .thenReturn(new Flags());
- when(mailboxMessage.getUid())
- .thenReturn(MESSAGE_UID);
-
- IndexableMessage indexableMessage = IndexableMessage.builder()
- .message(mailboxMessage)
- .extractor(new DefaultTextExtractor())
- .zoneId(ZoneId.of("Europe/Paris"))
- .indexAttachments(IndexAttachments.NO)
- .build();
-
- assertThat(indexableMessage.getText()).isEqualTo("My body");
- }
-
- @Test
- void textShouldContainsAllFieldsWhenAllSet() throws Exception {
- MailboxMessage mailboxMessage = mock(MailboxMessage.class);
- TestId mailboxId = TestId.of(1);
- when(mailboxMessage.getMailboxId())
- .thenReturn(mailboxId);
- when(mailboxMessage.getMessageId())
- .thenReturn(InMemoryMessageId.of(42));
- when(mailboxMessage.getModSeq())
- .thenReturn(ModSeq.first());
- when(mailboxMessage.getFullContent())
- .thenReturn(ClassLoader.getSystemResourceAsStream("eml/mailWithHeaders.eml"));
- when(mailboxMessage.createFlags())
- .thenReturn(new Flags());
- when(mailboxMessage.getUid())
- .thenReturn(MESSAGE_UID);
-
- IndexableMessage indexableMessage = IndexableMessage.builder()
- .message(mailboxMessage)
- .extractor(new DefaultTextExtractor())
- .zoneId(ZoneId.of("Europe/Paris"))
- .indexAttachments(IndexAttachments.NO)
- .build();
-
- assertThat(indexableMessage.getText()).isEqualTo("Ad Min admin@opush.test " +
- "a@test a@test B b@test " +
- "c@test c@test " +
- "dD d@test " +
- "my subject " +
- "Mail content\n" +
- "\n" +
- "-- \n" +
- "Ad Min\n");
- }
-
- @Test
void hasAttachmentsShouldReturnTrueWhenNonInlined() throws IOException {
//Given
MailboxMessage mailboxMessage = mock(MailboxMessage.class);
@@ -477,36 +243,6 @@ class IndexableMessageTest {
}
@Test
- void messageShouldBeIndexedEvenIfTikaParserThrowsAnError() throws Exception {
- //Given
- MailboxMessage mailboxMessage = mock(MailboxMessage.class);
- TestId mailboxId = TestId.of(1);
- when(mailboxMessage.getMailboxId())
- .thenReturn(mailboxId);
- when(mailboxMessage.getModSeq())
- .thenReturn(ModSeq.first());
- when(mailboxMessage.getMessageId())
- .thenReturn(InMemoryMessageId.of(42));
- when(mailboxMessage.getFullContent())
- .thenReturn(ClassLoader.getSystemResourceAsStream("eml/bodyMakeTikaToFail.eml"));
- when(mailboxMessage.createFlags())
- .thenReturn(new Flags());
- when(mailboxMessage.getUid())
- .thenReturn(MESSAGE_UID);
-
- // When
- IndexableMessage indexableMessage = IndexableMessage.builder()
- .message(mailboxMessage)
- .extractor(textExtractor)
- .zoneId(ZoneId.of("Europe/Paris"))
- .indexAttachments(IndexAttachments.YES)
- .build();
-
- // Then
- assertThat(indexableMessage.getText()).contains("subject should be parsed");
- }
-
- @Test
void shouldHandleCorrectlyMessageIdHavingSerializeMethodThatReturnNull() throws Exception {
MessageId invalidMessageIdThatReturnNull = mock(MessageId.class);
when(invalidMessageIdThatReturnNull.serialize())
diff --git a/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndexTest.java b/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndexTest.java
index 2124af0..34f767c 100644
--- a/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndexTest.java
+++ b/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndexTest.java
@@ -210,26 +210,6 @@ class SimpleMessageSearchIndexTest extends AbstractMessageSearchIndexTest {
public void multimailboxSearchShouldReturnUidOfMessageWithExpectedFromInTwoMailboxes() {
}
- @Disabled("JAMES-1804: failing jmap text search test on memory (intended for ES)")
- @Override
- public void searchWithTextShouldReturnMailsWhenTextBodyMatchesAndNonContinuousWords() {
- }
-
- @Disabled("JAMES-1804: failing jmap text search test on memory (intended for ES)")
- @Override
- public void searchWithTextShouldReturnMailsWhenHtmlBodyMatchesAndNonContinuousWords() {
- }
-
- @Disabled("JAMES-1804: failing jmap text search test on memory (intended for ES)")
- @Override
- public void searchWithTextShouldReturnMailsWhenTextBodyWithExtraUnindexedWords() {
- }
-
- @Disabled("JAMES-1804: failing jmap text search test on memory (intended for ES)")
- @Override
- public void searchWithTextShouldReturnMailsWhenHtmlBodyMatchesWithStemming() {
- }
-
@Disabled("JAMES-2241: memory does not handle header with dots indexation (intended for ES)")
@Override
public void headerWithDotsShouldBeIndexed() {
diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearches.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearches.java
index ca3f18c..45ce294 100644
--- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearches.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearches.java
@@ -23,7 +23,6 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.SequenceInputStream;
import java.io.StringReader;
import java.text.SimpleDateFormat;
import java.util.Arrays;
@@ -203,8 +202,6 @@ public class MessageSearches implements Iterable<SimpleMessageSearchIndex.Search
switch (criterion.getType()) {
case BODY:
return bodyContains(value, message);
- case TEXT:
- return textContains(value, message);
case FULL:
return messageContains(value, message);
case ATTACHMENTS:
@@ -238,11 +235,6 @@ public class MessageSearches implements Iterable<SimpleMessageSearchIndex.Search
return isInMessage(value, input, true);
}
- private boolean textContains(String value, MailboxMessage message) throws IOException, MimeException, MailboxException {
- InputStream bodyContent = message.getBodyContent();
- return isInMessage(value, new SequenceInputStream(textHeaders(message), bodyContent), true);
- }
-
private boolean attachmentsContain(String value, MailboxMessage message) throws IOException, MimeException {
List<MessageAttachmentMetadata> attachments = message.getAttachments();
return isInAttachments(value, attachments);
diff --git a/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/AbstractMessageSearchIndexTest.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/AbstractMessageSearchIndexTest.java
index 74b08f7..b85fdeb 100644
--- a/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/AbstractMessageSearchIndexTest.java
+++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/AbstractMessageSearchIndexTest.java
@@ -1233,147 +1233,6 @@ public abstract class AbstractMessageSearchIndexTest {
}
@Test
- void searchWithTextShouldReturnNoMailWhenNotMatching() throws Exception {
- assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.Text));
- SearchQuery searchQuery = SearchQuery.of(SearchQuery.textContains("unmatching"));
-
- assertThat(messageSearchIndex.search(session, mailbox, searchQuery))
- .isEmpty();
- }
-
- @Test
- void searchWithTextShouldReturnMailsWhenFromMatches() throws Exception {
- assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.Text));
- SearchQuery searchQuery = SearchQuery.of(SearchQuery.textContains("spam.minet.net"));
-
- assertThat(messageSearchIndex.search(session, mailbox, searchQuery))
- .containsExactly(m1.getUid());
- }
-
- @Test
- void searchWithTextShouldReturnMailsWhenMatches() throws Exception {
- assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.Text));
- SearchQuery searchQuery = SearchQuery.of(SearchQuery.textContains("root@listes.minet.net"));
-
- assertThat(messageSearchIndex.search(session, mailbox, searchQuery))
- .containsExactly(m1.getUid());
- }
-
- @Test
- void searchWithTextShouldReturnMailsWhenToIsNotAnExactMatches() throws Exception {
- assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.Text));
- SearchQuery searchQuery = SearchQuery.of(SearchQuery.textContains("root"));
-
- assertThat(messageSearchIndex.search(session, mailbox, searchQuery))
- .containsExactly(m1.getUid());
- }
-
- @Test
- void searchWithTextShouldReturnMailsWhenCcMatches() throws Exception {
- assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.Text));
- SearchQuery searchQuery = SearchQuery.of(SearchQuery.textContains("abc@abc.org"));
-
- assertThat(messageSearchIndex.search(session, mailbox, searchQuery))
- .containsExactly(m3.getUid());
- }
-
- @Test
- void searchWithTextShouldReturnMailsWhenCcIsNotAExactMatch() throws Exception {
- assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.Text));
- SearchQuery searchQuery = SearchQuery.of(SearchQuery.textContains("monkey"));
-
- assertThat(messageSearchIndex.search(session, mailbox, searchQuery))
- .containsExactly(m5.getUid());
- }
-
- @Test
- void searchWithTextShouldReturnMailsWhenBccMatches() throws Exception {
- assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.Text));
- SearchQuery searchQuery = SearchQuery.of(SearchQuery.textContains("monkey@any.com"));
-
- assertThat(messageSearchIndex.search(session, mailbox, searchQuery))
- .containsExactly(m5.getUid());
- }
-
- @Test
- void searchWithTextShouldReturnMailsWhenBccIsNotAExactMatch() throws Exception {
- assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.Text));
- SearchQuery searchQuery = SearchQuery.of(SearchQuery.textContains("monkey"));
-
- assertThat(messageSearchIndex.search(session, mailbox, searchQuery))
- .containsExactly(m5.getUid());
- }
-
- @Test
- void searchWithTextShouldReturnMailsWhenTextBodyMatches() throws Exception {
- assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.Text));
- SearchQuery searchQuery = SearchQuery.of(SearchQuery.textContains("reviewing work"));
-
- // text/plain contains: "We are reviewing work I did for this feature."
- assertThat(messageSearchIndex.search(session, mailbox, searchQuery))
- .containsExactly(m3.getUid());
- }
-
- @Test
- void searchWithTextShouldReturnMailsWhenTextBodyMatchesAndNonContinuousWords() throws Exception {
- assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.Text));
- SearchQuery searchQuery = SearchQuery.of(SearchQuery.textContains("reviewing feature"));
- // 2: text/plain contains: "Issue Type: New Feature"
- // 3: text/plain contains: "We are reviewing work I did for this feature."
-
- assertThat(messageSearchIndex.search(session, mailbox, searchQuery))
- .containsExactly(m2.getUid(), m3.getUid());
- }
-
- @Test
- void searchWithTextShouldReturnMailsWhenTextBodyMatchesInsensitiveWords() throws Exception {
- assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.Text));
- SearchQuery searchQuery = SearchQuery.of(SearchQuery.textContains("reVieWing"));
- // text/plain contains: "We are reviewing work I did for this feature."
-
- assertThat(messageSearchIndex.search(session, mailbox, searchQuery))
- .containsExactly(m3.getUid());
- }
-
- @Test
- void searchWithTextShouldReturnMailsWhenTextBodyWithExtraUnindexedWords() throws Exception {
- assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.Text));
- SearchQuery searchQuery = SearchQuery.of(SearchQuery.textContains("a reviewing of the work"));
- // text/plain contains: "We are reviewing work I did for this feature."
-
- assertThat(messageSearchIndex.search(session, mailbox, searchQuery))
- .containsExactly(m3.getUid());
- }
-
- @Test
- void searchWithTextShouldReturnMailsWhenHtmlBodyMatches() throws Exception {
- assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.Text));
- SearchQuery searchQuery = SearchQuery.of(SearchQuery.textContains("contains a banana"));
- // text/html contains: "This is a mail with beautifull html content which contains a banana."
-
- assertThat(messageSearchIndex.search(session, mailbox, searchQuery))
- .containsExactly(m7.getUid());
- }
-
- @Test
- void searchWithTextShouldReturnMailsWhenHtmlBodyMatchesWithStemming() throws Exception {
- assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.Text));
- SearchQuery searchQuery = SearchQuery.of(SearchQuery.textContains("contain banana"));
-
- assertThat(messageSearchIndex.search(session, mailbox, searchQuery))
- .containsExactly(m7.getUid());
- }
-
- @Test
- void searchWithTextShouldReturnMailsWhenHtmlBodyMatchesAndNonContinuousWords() throws Exception {
- assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.Text));
- SearchQuery searchQuery = SearchQuery.of(SearchQuery.textContains("beautifull banana"));
-
- assertThat(messageSearchIndex.search(session, mailbox, searchQuery))
- .containsExactly(m7.getUid());
- }
-
- @Test
void searchWithFullTextShouldReturnMailsWhenNotAPerfectMatch() throws Exception {
assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.FullText));
ComposedMessageId messageWithBeautifulBananaAsTextAttachment = myFolderMessageManager.appendMessage(
diff --git a/mailbox/store/src/test/resources/eml/emailWithNonIndexableAttachmentWithoutAttachment.json b/mailbox/store/src/test/resources/eml/emailWithNonIndexableAttachmentWithoutAttachment.json
index 76abe79..337c47e 100644
--- a/mailbox/store/src/test/resources/eml/emailWithNonIndexableAttachmentWithoutAttachment.json
+++ b/mailbox/store/src/test/resources/eml/emailWithNonIndexableAttachmentWithoutAttachment.json
@@ -16,7 +16,6 @@
"mimeMessageID": "1480502737913",
"bcc" : [],
"attachments" : [],
- "text" : "Laura ROYET laura.ro@linagora.com Laura ROYET laura.ro@linagora.com test <!DOCTYPE html>\n<html class=\"mozilla\" lang=\"en\"><head>\n<meta http-equiv=\"content-type\" content=\"text/html; charset=UTF-8\">\n \n\n\n\n\n\n<meta charset=\"utf-8\">\n<meta http-equiv=\"X-UA-Compatible\" content=\"IE=Edge\">\n<meta name=\"application-name\" content=\"JIRA\" data-name=\"jira\" data-version=\"6.4.5\"><meta name=\"ajs-serverRenderedViewIssue\" content=\"true\">\n<meta name=\"ajs-h [...]
"isFlagged" : false,
"mediaType" : "plain",
"hasAttachment" : false,
diff --git a/mailbox/store/src/test/resources/eml/htmlMail.json b/mailbox/store/src/test/resources/eml/htmlMail.json
index a07c128..326664b 100644
--- a/mailbox/store/src/test/resources/eml/htmlMail.json
+++ b/mailbox/store/src/test/resources/eml/htmlMail.json
@@ -141,6 +141,5 @@
"isFlagged":false,
"isRecent":false,
"hasAttachment":false,
- "isUnread":false,
- "text": "Airbnb discover@airbnb.com mister@james.org mister@james.org Text and Html not similar The text/plain part is not matching the html one.\r\n <html>\r\n <head>\r\n\r\n <meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\">\r\n </head>\r\n <body bgcolor=\"#FFFFFF\" text=\"#000000\">\r\n <i>This is a mail with <b>beautifull</b> html content which contains a banana.</i><br>\r\n </body>\r\n</html>\r\n"
+ "isUnread":false
}
diff --git a/mailbox/store/src/test/resources/eml/invalidCharset.json b/mailbox/store/src/test/resources/eml/invalidCharset.json
index eed4184..f9ead9f 100644
--- a/mailbox/store/src/test/resources/eml/invalidCharset.json
+++ b/mailbox/store/src/test/resources/eml/invalidCharset.json
@@ -25,7 +25,6 @@
"size":25,
"subject":["Inline attachment"],
"subtype":"text",
- "text":"Antoine DUPRAT xyz@linagora.com Antoine DUPRAT xyz@linagora.com Inline attachment This is an inline attachment: Cheers!",
"to":[{"name":"Antoine DUPRAT","address":"xyz@linagora.com"}],
"uid":25,
"userFlags":[],
diff --git a/mailbox/store/src/test/resources/eml/mail.json b/mailbox/store/src/test/resources/eml/mail.json
index 4485621..e1d526d 100644
--- a/mailbox/store/src/test/resources/eml/mail.json
+++ b/mailbox/store/src/test/resources/eml/mail.json
@@ -170,6 +170,5 @@
"isDeleted": true,
"isRecent": false,
"hasAttachment": false,
- "isUnread": false,
- "text": "Murari murari.ksr@gmail.com General Discussion about Arch Linux arch-general@archlinux.org [arch-general] Inkscape fails to open svg files Hi all,\n\nAn update between the 17th of May and the 1st of June seems to have broken\nInkscape's ability to open existing svg files on my computer. Any svg I\nopen (including stock files off the internet) cause Inkscape to crash with\n\"Inkscape encountered an internal error and will close now\". I even created\na new, test svg consisting o [...]
+ "isUnread": false
}
diff --git a/mailbox/store/src/test/resources/eml/nonTextual.json b/mailbox/store/src/test/resources/eml/nonTextual.json
index 06a12dd..48d5d15 100644
--- a/mailbox/store/src/test/resources/eml/nonTextual.json
+++ b/mailbox/store/src/test/resources/eml/nonTextual.json
@@ -68,7 +68,6 @@
"size":25,
"subject":["Test message"],
"subtype":"text",
- "text":"Benoit Tellier btellier@linagora.com btellier@linagora.com btellier@linagora.com Test message This mail have a non textual attachment !\r\n",
"to":[{"name":"btellier@linagora.com","address":"btellier@linagora.com"}],
"uid":25,
"userFlags":[],
diff --git a/mailbox/store/src/test/resources/eml/pgpSignedMail.json b/mailbox/store/src/test/resources/eml/pgpSignedMail.json
index 2b6bf5d..f5673d5 100644
--- a/mailbox/store/src/test/resources/eml/pgpSignedMail.json
+++ b/mailbox/store/src/test/resources/eml/pgpSignedMail.json
@@ -169,6 +169,5 @@
"isFlagged": false,
"isRecent": false,
"hasAttachment": false,
- "isUnread": false,
- "text": "Markus Koschany apo@gambaru.de debian-security-announce@lists.debian.org debian-security-announce@lists.debian.org [SECURITY] [DSA 3278-1] libapache-mod-jk security update -----BEGIN PGP SIGNED MESSAGE-----\r\nHash: SHA512\r\n\r\n- -------------------------------------------------------------------------\r\nDebian Security Advisory DSA-3278-1 security@debian.org\r\nhttp://www.debian.org/security/ Markus Koschany\r\nJune 03, 2015 [...]
+ "isUnread": false
}
diff --git a/mailbox/store/src/test/resources/eml/recursiveMail.json b/mailbox/store/src/test/resources/eml/recursiveMail.json
index ddfc39c..ba6edc4 100644
--- a/mailbox/store/src/test/resources/eml/recursiveMail.json
+++ b/mailbox/store/src/test/resources/eml/recursiveMail.json
@@ -126,6 +126,5 @@
"isAnswered": false,
"isDeleted": true,
"hasAttachment": true,
- "isUnread": false,
- "text": "Matthieu Baechler mbaechler@linagora.com btellier@linagora.com btellier@linagora.com Fwd: Courbe Sprint Forward as attachment !\n\n\n"
+ "isUnread": false
}
diff --git a/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json b/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json
index ccdfb83..b0f7c5d 100644
--- a/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json
+++ b/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json
@@ -85,6 +85,5 @@
"isAnswered": false,
"isDeleted": true,
"hasAttachment": true,
- "isUnread": false,
- "text": "Matthieu Baechler mbaechler@linagora.com btellier@linagora.com btellier@linagora.com Fwd: Courbe Sprint Forward as attachment !\n\n\n"
+ "isUnread": false
}
diff --git a/mailbox/store/src/test/resources/eml/spamMail.json b/mailbox/store/src/test/resources/eml/spamMail.json
index 5209c7a..626881a 100644
--- a/mailbox/store/src/test/resources/eml/spamMail.json
+++ b/mailbox/store/src/test/resources/eml/spamMail.json
@@ -149,6 +149,5 @@
"isFlagged": false,
"isRecent": false,
"hasAttachment": false,
- "isUnread": true,
- "text": "Content-filter at spam.minet.net postmaster@minet.net root@listes.minet.net root@listes.minet.net [root] UNCHECKED contents in mail FROM <qu...@riseup.net> No viruses were found.\r\n\r\nContent type: Unchecked\r\nInternal reference code for the message is 12583-16/Ss-IZaXqmZao\r\n\r\nAccording to a 'Received:' trace, the message apparently originated at:\r\n [198.252.153.129], [127.0.0.1] localhost [127.0.0.1] Authenticated sender:\r\n quentin.h\r\n\r\nReturn-Path: <quen [...]
+ "isUnread": true
}
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org