You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2017/06/28 08:48:31 UTC
[2/3] james-project git commit: MAILBOX-301 Add tests for terms
exceeding Lucene size
MAILBOX-301 Add tests for terms exceeding Lucene size
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/829adc7c
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/829adc7c
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/829adc7c
Branch: refs/heads/master
Commit: 829adc7c3a36fc643216e480021bc284a7a27f57
Parents: e341b02
Author: benwa <bt...@linagora.com>
Authored: Tue Jun 27 13:03:53 2017 +0700
Committer: benwa <bt...@linagora.com>
Committed: Wed Jun 28 15:47:10 2017 +0700
----------------------------------------------------------------------
.../ElasticSearchIntegrationTest.java | 105 +++++++++++++++++++
.../search/AbstractMessageSearchIndexTest.java | 6 +-
2 files changed, 108 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/829adc7c/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java
index 2495394..e690f95 100644
--- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java
+++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java
@@ -19,15 +19,22 @@
package org.apache.james.mailbox.elasticsearch;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.ByteArrayInputStream;
import java.time.ZoneId;
+import java.util.Date;
import java.util.concurrent.Executors;
+import javax.mail.Flags;
+
import org.apache.james.backends.es.DeleteByQueryPerformer;
import org.apache.james.backends.es.ElasticSearchIndexer;
import org.apache.james.backends.es.EmbeddedElasticSearch;
import org.apache.james.backends.es.IndexCreationFactory;
import org.apache.james.backends.es.NodeMappingFactory;
import org.apache.james.backends.es.utils.TestingClientProvider;
+import org.apache.james.mailbox.MessageManager;
import org.apache.james.mailbox.acl.SimpleGroupMembershipResolver;
import org.apache.james.mailbox.acl.UnionMailboxACLResolver;
import org.apache.james.mailbox.elasticsearch.events.ElasticSearchListeningMessageSearchIndex;
@@ -40,6 +47,11 @@ import org.apache.james.mailbox.inmemory.InMemoryMailboxManager;
import org.apache.james.mailbox.inmemory.InMemoryMailboxSessionMapperFactory;
import org.apache.james.mailbox.inmemory.InMemoryMessageId;
import org.apache.james.mailbox.inmemory.InMemoryMessageIdManager;
+import org.apache.james.mailbox.mock.MockMailboxSession;
+import org.apache.james.mailbox.model.ComposedMessageId;
+import org.apache.james.mailbox.model.MailboxConstants;
+import org.apache.james.mailbox.model.MailboxPath;
+import org.apache.james.mailbox.model.SearchQuery;
import org.apache.james.mailbox.store.FakeAuthenticator;
import org.apache.james.mailbox.store.FakeAuthorizator;
import org.apache.james.mailbox.store.JVMMailboxPathLocker;
@@ -48,14 +60,20 @@ import org.apache.james.mailbox.store.extractor.DefaultTextExtractor;
import org.apache.james.mailbox.store.mail.model.impl.MessageParser;
import org.apache.james.mailbox.store.search.AbstractMessageSearchIndexTest;
import org.elasticsearch.client.Client;
+import org.junit.Ignore;
import org.junit.Rule;
+import org.junit.Test;
import org.junit.rules.RuleChain;
import org.junit.rules.TemporaryFolder;
+import com.google.common.base.Charsets;
+import com.google.common.base.Strings;
+
public class ElasticSearchIntegrationTest extends AbstractMessageSearchIndexTest {
private static final int BATCH_SIZE = 1;
private static final int SEARCH_SIZE = 1;
+ private static final boolean IS_RECENT = true;
private TemporaryFolder temporaryFolder = new TemporaryFolder();
private EmbeddedElasticSearch embeddedElasticSearch= new EmbeddedElasticSearch(temporaryFolder, MailboxElasticsearchConstants.MAILBOX_INDEX);
@@ -104,4 +122,91 @@ public class ElasticSearchIntegrationTest extends AbstractMessageSearchIndexTest
storeMailboxManager.setMessageSearchIndex(messageSearchIndex);
storeMailboxManager.init();
}
+
+ @Ignore("MAILBOX-301 Lucene terms length exceeded on some emails")
+ @Test
+ public void termsBetweenElasticSearchAndLuceneLimitDueTuNonAsciiCharsShouldBeTruncated() throws Exception {
+ MailboxPath mailboxPath = new MailboxPath(MailboxConstants.USER_NAMESPACE, USERNAME, INBOX);
+ MockMailboxSession session = new MockMailboxSession(USERNAME);
+ MessageManager messageManager = storeMailboxManager.getMailbox(mailboxPath, session);
+
+ String recipient = "benwa@linagora.com";
+ ComposedMessageId composedMessageId = messageManager.appendMessage(new ByteArrayInputStream(("To: " + recipient + "\n" +
+ "\n" +
+ Strings.repeat("0à2345678é", 3200)).getBytes(Charsets.UTF_8)), new Date(), session, IS_RECENT, new Flags());
+
+ embeddedElasticSearch.awaitForElasticSearch();
+
+ assertThat(messageManager.search(new SearchQuery(SearchQuery.address(SearchQuery.AddressType.To, recipient)), session))
+ .containsExactly(composedMessageId.getUid());
+ }
+
+ @Test
+ public void tooLongTermsShouldNotMakeIndexingFail() throws Exception {
+ MailboxPath mailboxPath = new MailboxPath(MailboxConstants.USER_NAMESPACE, USERNAME, INBOX);
+ MockMailboxSession session = new MockMailboxSession(USERNAME);
+ MessageManager messageManager = storeMailboxManager.getMailbox(mailboxPath, session);
+
+ String recipient = "benwa@linagora.com";
+ ComposedMessageId composedMessageId = messageManager.appendMessage(new ByteArrayInputStream(("To: " + recipient + "\n" +
+ "\n" +
+ Strings.repeat("0123456789", 3300)).getBytes(Charsets.UTF_8)), new Date(), session, IS_RECENT, new Flags());
+
+ embeddedElasticSearch.awaitForElasticSearch();
+
+ assertThat(messageManager.search(new SearchQuery(SearchQuery.address(SearchQuery.AddressType.To, recipient)), session))
+ .containsExactly(composedMessageId.getUid());
+ }
+
+ @Test
+ public void fieldsExceedingLuceneLimitShouldNotBeIgnored() throws Exception {
+ MailboxPath mailboxPath = new MailboxPath(MailboxConstants.USER_NAMESPACE, USERNAME, INBOX);
+ MockMailboxSession session = new MockMailboxSession(USERNAME);
+ MessageManager messageManager = storeMailboxManager.getMailbox(mailboxPath, session);
+
+ String recipient = "benwa@linagora.com";
+ ComposedMessageId composedMessageId = messageManager.appendMessage(new ByteArrayInputStream(("To: " + recipient + "\n" +
+ "\n" +
+ Strings.repeat("0123456789 ", 5000)).getBytes(Charsets.UTF_8)), new Date(), session, IS_RECENT, new Flags());
+
+ embeddedElasticSearch.awaitForElasticSearch();
+
+ assertThat(messageManager.search(new SearchQuery(SearchQuery.bodyContains("0123456789")), session))
+ .containsExactly(composedMessageId.getUid());
+ }
+
+ @Test
+ public void fieldsWithTooLongTermShouldStillBeIndexed() throws Exception {
+ MailboxPath mailboxPath = new MailboxPath(MailboxConstants.USER_NAMESPACE, USERNAME, INBOX);
+ MockMailboxSession session = new MockMailboxSession(USERNAME);
+ MessageManager messageManager = storeMailboxManager.getMailbox(mailboxPath, session);
+
+ String recipient = "benwa@linagora.com";
+ ComposedMessageId composedMessageId = messageManager.appendMessage(new ByteArrayInputStream(("To: " + recipient + "\n" +
+ "\n" +
+ Strings.repeat("0123456789", 5000) + " matchMe").getBytes(Charsets.UTF_8)), new Date(), session, IS_RECENT, new Flags());
+
+ embeddedElasticSearch.awaitForElasticSearch();
+
+ assertThat(messageManager.search(new SearchQuery(SearchQuery.bodyContains("matchMe")), session))
+ .containsExactly(composedMessageId.getUid());
+ }
+
+ @Test
+ public void reasonableLongTermShouldNotBeIgnored() throws Exception {
+ MailboxPath mailboxPath = new MailboxPath(MailboxConstants.USER_NAMESPACE, USERNAME, INBOX);
+ MockMailboxSession session = new MockMailboxSession(USERNAME);
+ MessageManager messageManager = storeMailboxManager.getMailbox(mailboxPath, session);
+
+ String recipient = "benwa@linagora.com";
+ String reasonableLongTerm = "dichlorodiphényltrichloroéthane";
+ ComposedMessageId composedMessageId = messageManager.appendMessage(new ByteArrayInputStream(("To: " + recipient + "\n" +
+ "\n" +
+ reasonableLongTerm).getBytes(Charsets.UTF_8)), new Date(), session, IS_RECENT, new Flags());
+
+ embeddedElasticSearch.awaitForElasticSearch();
+
+ assertThat(messageManager.search(new SearchQuery(SearchQuery.bodyContains(reasonableLongTerm)), session))
+ .containsExactly(composedMessageId.getUid());
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/james-project/blob/829adc7c/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/AbstractMessageSearchIndexTest.java
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/AbstractMessageSearchIndexTest.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/AbstractMessageSearchIndexTest.java
index c489d80..8c45ae0 100644
--- a/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/AbstractMessageSearchIndexTest.java
+++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/AbstractMessageSearchIndexTest.java
@@ -54,9 +54,9 @@ import com.google.common.collect.ImmutableList;
public abstract class AbstractMessageSearchIndexTest {
- private static final String INBOX = "INBOX";
- private static final String OTHERUSER = "otheruser";
- private static final String USERNAME = "benwa";
+ protected static final String INBOX = "INBOX";
+ protected static final String OTHERUSER = "otheruser";
+ protected static final String USERNAME = "benwa";
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractMessageSearchIndexTest.class);
public static final long LIMIT = 100L;
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org