You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2018/08/03 08:52:21 UTC
[08/20] james-project git commit: JAMES-2517 Use try-with-resource in
more places
http://git-wip-us.apache.org/repos/asf/james-project/blob/3c275fe6/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
----------------------------------------------------------------------
diff --git a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
index 887b05a..13a402d 100644
--- a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
+++ b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
@@ -1,1383 +1,1351 @@
-/****************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one *
- * or more contributor license agreements. See the NOTICE file *
- * distributed with this work for additional information *
- * regarding copyright ownership. The ASF licenses this file *
- * to you under the Apache License, Version 2.0 (the *
- * "License"); you may not use this file except in compliance *
- * with the License. You may obtain a copy of the License at *
- * *
- * http://www.apache.org/licenses/LICENSE-2.0 *
- * *
- * Unless required by applicable law or agreed to in writing, *
- * software distributed under the License is distributed on an *
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
- * KIND, either express or implied. See the License for the *
- * specific language governing permissions and limitations *
- * under the License. *
- ****************************************************************/
-package org.apache.james.mailbox.lucene.search;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.StringReader;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Collection;
-import java.util.Date;
-import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Optional;
-import java.util.Set;
-import java.util.TimeZone;
-
-import javax.inject.Inject;
-import javax.mail.Flags;
-import javax.mail.Flags.Flag;
-
-import org.apache.james.mailbox.MailboxManager;
-import org.apache.james.mailbox.MailboxManager.SearchCapabilities;
-import org.apache.james.mailbox.MailboxSession;
-import org.apache.james.mailbox.MessageUid;
-import org.apache.james.mailbox.exception.MailboxException;
-import org.apache.james.mailbox.exception.UnsupportedSearchException;
-import org.apache.james.mailbox.model.MailboxId;
-import org.apache.james.mailbox.model.MessageId;
-import org.apache.james.mailbox.model.MessageRange;
-import org.apache.james.mailbox.model.SearchQuery;
-import org.apache.james.mailbox.model.SearchQuery.AllCriterion;
-import org.apache.james.mailbox.model.SearchQuery.AttachmentCriterion;
-import org.apache.james.mailbox.model.SearchQuery.ContainsOperator;
-import org.apache.james.mailbox.model.SearchQuery.Criterion;
-import org.apache.james.mailbox.model.SearchQuery.CustomFlagCriterion;
-import org.apache.james.mailbox.model.SearchQuery.DateOperator;
-import org.apache.james.mailbox.model.SearchQuery.DateResolution;
-import org.apache.james.mailbox.model.SearchQuery.FlagCriterion;
-import org.apache.james.mailbox.model.SearchQuery.HeaderCriterion;
-import org.apache.james.mailbox.model.SearchQuery.HeaderOperator;
-import org.apache.james.mailbox.model.SearchQuery.NumericOperator;
-import org.apache.james.mailbox.model.SearchQuery.UidCriterion;
-import org.apache.james.mailbox.model.SearchQuery.UidRange;
-import org.apache.james.mailbox.model.UpdatedFlags;
-import org.apache.james.mailbox.store.mail.MessageMapperFactory;
-import org.apache.james.mailbox.store.mail.model.Mailbox;
-import org.apache.james.mailbox.store.mail.model.MailboxMessage;
-import org.apache.james.mailbox.store.mail.model.impl.PropertyBuilder;
-import org.apache.james.mailbox.store.search.ListeningMessageSearchIndex;
-import org.apache.james.mailbox.store.search.SearchUtil;
-import org.apache.james.mime4j.MimeException;
-import org.apache.james.mime4j.dom.Header;
-import org.apache.james.mime4j.dom.address.Address;
-import org.apache.james.mime4j.dom.address.AddressList;
-import org.apache.james.mime4j.dom.address.Group;
-import org.apache.james.mime4j.dom.address.MailboxList;
-import org.apache.james.mime4j.dom.datetime.DateTime;
-import org.apache.james.mime4j.dom.field.DateTimeField;
-import org.apache.james.mime4j.field.address.AddressFormatter;
-import org.apache.james.mime4j.field.address.LenientAddressParser;
-import org.apache.james.mime4j.field.datetime.parser.DateTimeParser;
-import org.apache.james.mime4j.message.SimpleContentHandler;
-import org.apache.james.mime4j.parser.MimeStreamParser;
-import org.apache.james.mime4j.stream.BodyDescriptor;
-import org.apache.james.mime4j.stream.MimeConfig;
-import org.apache.james.mime4j.util.MimeUtil;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.DateTools;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.NumericField;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.NumericRangeQuery;
-import org.apache.lucene.search.PrefixQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TermRangeQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.WildcardQuery;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.Version;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.github.steveash.guavate.Guavate;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-
-/**
- * Lucene based {@link ListeningMessageSearchIndex} which offers message searching via a Lucene index
- *
- *
-
- * @param
- */
-public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
- private static final Logger LOGGER = LoggerFactory.getLogger(LuceneMessageSearchIndex.class);
- private static final Date MAX_DATE;
- private static final Date MIN_DATE;
-
- static {
- Calendar cal = Calendar.getInstance();
- cal.set(9999, 11, 31);
- MAX_DATE = cal.getTime();
-
- cal.set(0000, 0, 1);
- MIN_DATE = cal.getTime();
- }
-
- /**
- * Default max query results
- */
- private static final int DEFAULT_MAX_QUERY_RESULTS = 100000;
-
- /**
- * {@link Field} which will contain the unique index of the {@link Document}
- */
- private static final String ID_FIELD = "id";
-
-
- /**
- * {@link Field} which will contain uid of the {@link MailboxMessage}
- */
- private static final String UID_FIELD = "uid";
-
- /**
- * {@link Field} boolean field that say if the message as an attachment or not
- */
- private static final String HAS_ATTACHMENT_FIELD = "hasAttachment";
-
- /**
- * {@link Field} which will contain the {@link Flags} of the {@link MailboxMessage}
- */
- private static final String FLAGS_FIELD = "flags";
-
- /**
- * {@link Field} which will contain the size of the {@link MailboxMessage}
- */
- private static final String SIZE_FIELD = "size";
-
- /**
- * {@link Field} which will contain the body of the {@link MailboxMessage}
- */
- private static final String BODY_FIELD = "body";
-
-
- /**
- * Prefix which will be used for each message header to store it also in a seperate {@link Field}
- */
- private static final String PREFIX_HEADER_FIELD = "header_";
-
- /**
- * {@link Field} which will contain the whole message header of the {@link MailboxMessage}
- */
- private static final String HEADERS_FIELD = "headers";
-
- /**
- * {@link Field} which will contain the mod-sequence of the message
- */
- private static final String MODSEQ_FIELD = "modSeq";
-
- /**
- * {@link Field} which will contain the TO-Address of the message
- */
- private static final String TO_FIELD = "to";
-
- private static final String FIRST_TO_MAILBOX_NAME_FIELD = "firstToMailboxName";
- private static final String FIRST_TO_MAILBOX_DISPLAY_FIELD = "firstToMailboxDisplay";
-
- /**
- * {@link Field} which will contain the CC-Address of the message
- */
- private static final String CC_FIELD = "cc";
-
- private static final String FIRST_CC_MAILBOX_NAME_FIELD = "firstCcMailboxName";
-
-
- /**
- * {@link Field} which will contain the FROM-Address of the message
- */
- private static final String FROM_FIELD = "from";
-
- private static final String FIRST_FROM_MAILBOX_NAME_FIELD = "firstFromMailboxName";
- private static final String FIRST_FROM_MAILBOX_DISPLAY_FIELD = "firstFromMailboxDisplay";
-
- /**
- * {@link Field} which will contain the BCC-Address of the message
- */
- private static final String BCC_FIELD = "bcc";
-
-
- private static final String BASE_SUBJECT_FIELD = "baseSubject";
-
- /**
- * {@link Field} which contain the internalDate of the message with YEAR-Resolution
- */
- private static final String INTERNAL_DATE_FIELD_YEAR_RESOLUTION = "internaldateYearResolution";
-
-
- /**
- * {@link Field} which contain the internalDate of the message with MONTH-Resolution
- */
- private static final String INTERNAL_DATE_FIELD_MONTH_RESOLUTION = "internaldateMonthResolution";
-
- /**
- * {@link Field} which contain the internalDate of the message with DAY-Resolution
- */
- private static final String INTERNAL_DATE_FIELD_DAY_RESOLUTION = "internaldateDayResolution";
-
- /**
- * {@link Field} which contain the internalDate of the message with HOUR-Resolution
- */
- private static final String INTERNAL_DATE_FIELD_HOUR_RESOLUTION = "internaldateHourResolution";
-
- /**
- * {@link Field} which contain the internalDate of the message with MINUTE-Resolution
- */
- private static final String INTERNAL_DATE_FIELD_MINUTE_RESOLUTION = "internaldateMinuteResolution";
-
- /**
- * {@link Field} which contain the internalDate of the message with SECOND-Resolution
- */
- private static final String INTERNAL_DATE_FIELD_SECOND_RESOLUTION = "internaldateSecondResolution";
-
-
- /**
- * {@link Field} which contain the internalDate of the message with MILLISECOND-Resolution
- */
- private static final String INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION = "internaldateMillisecondResolution";
-
- /**
- * {@link Field} which will contain the id of the {@link Mailbox}
- */
- private static final String MAILBOX_ID_FIELD = "mailboxid";
-
- /**
- * {@link Field} which will contain the user of the {@link MailboxSession}
- */
- private static final String USERS = "userSession";
- /**
- * {@link Field} which will contain the id of the {@link MessageId}
- */
- private static final String MESSAGE_ID_FIELD = "messageid";
-
- /**
- * {@link Field} which contain the Date header of the message with YEAR-Resolution
- */
- private static final String SENT_DATE_FIELD_YEAR_RESOLUTION = "sentdateYearResolution";
-
-
- /**
- * {@link Field} which contain the Date header of the message with MONTH-Resolution
- */
- private static final String SENT_DATE_FIELD_MONTH_RESOLUTION = "sentdateMonthResolution";
-
- /**
- * {@link Field} which contain the Date header of the message with DAY-Resolution
- */
- private static final String SENT_DATE_FIELD_DAY_RESOLUTION = "sentdateDayResolution";
-
- /**
- * {@link Field} which contain the Date header of the message with HOUR-Resolution
- */
- private static final String SENT_DATE_FIELD_HOUR_RESOLUTION = "sentdateHourResolution";
-
- /**
- * {@link Field} which contain the Date header of the message with MINUTE-Resolution
- */
- private static final String SENT_DATE_FIELD_MINUTE_RESOLUTION = "sentdateMinuteResolution";
-
- /**
- * {@link Field} which contain the Date header of the message with SECOND-Resolution
- */
- private static final String SENT_DATE_FIELD_SECOND_RESOLUTION = "sentdateSecondResolution";
-
-
- /**
- * {@link Field} which contain the Date header of the message with MILLISECOND-Resolution
- */
- private static final String SENT_DATE_FIELD_MILLISECOND_RESOLUTION = "sentdateMillisecondResolution";
-
- private static final String SENT_DATE_SORT_FIELD_MILLISECOND_RESOLUTION = "sentdateSort";
-
- private static final String MEDIA_TYPE_TEXT = "text";
- private static final String MEDIA_TYPE_MESSAGE = "message";
- private static final String DEFAULT_ENCODING = "US-ASCII";
-
- private static final SortField UID_SORT = new SortField(UID_FIELD, SortField.LONG);
- private static final SortField UID_SORT_REVERSE = new SortField(UID_FIELD, SortField.LONG, true);
-
- private static final SortField SIZE_SORT = new SortField(SIZE_FIELD, SortField.LONG);
- private static final SortField SIZE_SORT_REVERSE = new SortField(SIZE_FIELD, SortField.LONG, true);
-
- private static final SortField FIRST_CC_MAILBOX_SORT = new SortField(FIRST_CC_MAILBOX_NAME_FIELD, SortField.STRING);
- private static final SortField FIRST_CC_MAILBOX_SORT_REVERSE = new SortField(FIRST_CC_MAILBOX_NAME_FIELD, SortField.STRING, true);
-
- private static final SortField FIRST_TO_MAILBOX_SORT = new SortField(FIRST_TO_MAILBOX_NAME_FIELD, SortField.STRING);
- private static final SortField FIRST_TO_MAILBOX_SORT_REVERSE = new SortField(FIRST_TO_MAILBOX_NAME_FIELD, SortField.STRING, true);
-
- private static final SortField FIRST_FROM_MAILBOX_SORT = new SortField(FIRST_FROM_MAILBOX_NAME_FIELD, SortField.STRING);
- private static final SortField FIRST_FROM_MAILBOX_SORT_REVERSE = new SortField(FIRST_FROM_MAILBOX_NAME_FIELD, SortField.STRING, true);
-
-
- private static final SortField ARRIVAL_MAILBOX_SORT = new SortField(INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION, SortField.LONG);
- private static final SortField ARRIVAL_MAILBOX_SORT_REVERSE = new SortField(INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION, SortField.LONG, true);
-
- private static final SortField BASE_SUBJECT_SORT = new SortField(BASE_SUBJECT_FIELD, SortField.STRING);
- private static final SortField BASE_SUBJECT_SORT_REVERSE = new SortField(BASE_SUBJECT_FIELD, SortField.STRING, true);
-
- private static final SortField SENT_DATE_SORT = new SortField(SENT_DATE_SORT_FIELD_MILLISECOND_RESOLUTION, SortField.LONG);
- private static final SortField SENT_DATE_SORT_REVERSE = new SortField(SENT_DATE_SORT_FIELD_MILLISECOND_RESOLUTION, SortField.LONG, true);
-
- private static final SortField FIRST_TO_MAILBOX_DISPLAY_SORT = new SortField(FIRST_TO_MAILBOX_DISPLAY_FIELD, SortField.STRING);
- private static final SortField FIRST_TO_MAILBOX_DISPLAY_SORT_REVERSE = new SortField(FIRST_TO_MAILBOX_DISPLAY_FIELD, SortField.STRING, true);
-
- private static final SortField FIRST_FROM_MAILBOX_DISPLAY_SORT = new SortField(FIRST_FROM_MAILBOX_DISPLAY_FIELD, SortField.STRING);
- private static final SortField FIRST_FROM_MAILBOX_DISPLAY_SORT_REVERSE = new SortField(FIRST_FROM_MAILBOX_DISPLAY_FIELD, SortField.STRING, true);
-
- private final MailboxId.Factory mailboxIdFactory;
- private final MessageId.Factory messageIdFactory;
- private final IndexWriter writer;
-
- private int maxQueryResults = DEFAULT_MAX_QUERY_RESULTS;
-
- private boolean suffixMatch = false;
-
- @Inject
- public LuceneMessageSearchIndex(
- MessageMapperFactory factory,
- MailboxId.Factory mailboxIdFactory,
- Directory directory,
- MessageId.Factory messageIdFactory
- ) throws IOException {
- this(factory, mailboxIdFactory, directory, false, true, messageIdFactory);
- }
-
- public LuceneMessageSearchIndex(
- MessageMapperFactory factory,
- MailboxId.Factory mailboxIdFactory,
- Directory directory,
- boolean dropIndexOnStart,
- boolean lenient,
- MessageId.Factory messageIdFactory
- ) throws IOException {
- super(factory);
- this.mailboxIdFactory = mailboxIdFactory;
- this.messageIdFactory = messageIdFactory;
- this.writer = new IndexWriter(directory, createConfig(createAnalyzer(lenient), dropIndexOnStart));
- }
-
-
- public LuceneMessageSearchIndex(MessageMapperFactory factory, MailboxId.Factory mailboxIdFactory, MessageId.Factory messageIdFactory, IndexWriter writer) {
- super(factory);
- this.mailboxIdFactory = mailboxIdFactory;
- this.messageIdFactory = messageIdFactory;
- this.writer = writer;
- }
-
- @Override
- public ListenerType getType() {
- return ListenerType.EACH_NODE;
- }
-
-
- @Override
- public EnumSet<SearchCapabilities> getSupportedCapabilities(EnumSet<MailboxManager.MessageCapabilities> messageCapabilities) {
- return EnumSet.of(SearchCapabilities.MultimailboxSearch);
-
- }
-
- /**
- * Set the max count of results which will get returned from a query. The default is {@link #DEFAULT_MAX_QUERY_RESULTS}
- *
- * @param maxQueryResults
- */
- public void setMaxQueryResults(int maxQueryResults) {
- this.maxQueryResults = maxQueryResults;
- }
-
- protected IndexWriterConfig createConfig(Analyzer analyzer, boolean dropIndexOnStart) {
- IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_31, analyzer);
- if (dropIndexOnStart) {
- config.setOpenMode(OpenMode.CREATE);
- } else {
- config.setOpenMode(OpenMode.CREATE_OR_APPEND);
- }
- return config;
- }
-
- /**
- * Create a {@link Analyzer} which is used to index the {@link MailboxMessage}'s
- *
- * @param lenient
- *
- * @return analyzer
- */
- protected Analyzer createAnalyzer(boolean lenient) {
- if (lenient) {
- return new LenientImapSearchAnalyzer();
- } else {
- return new StrictImapSearchAnalyzer();
- }
- }
-
- /**
- * If set to true this implementation will use {@link WildcardQuery} to match suffix and prefix. This is what RFC3501 expects but is often not what the user does.
- * It also slow things a lot if you have complex queries which use many "TEXT" arguments. If you want the implementation to behave strict like RFC3501 says, you should
- * set this to true.
- *
- * The default is false for performance reasons
- *
- *
- * @param suffixMatch
- */
- public void setEnableSuffixMatch(boolean suffixMatch) {
- this.suffixMatch = suffixMatch;
- }
-
-
-
- @Override
- public Iterator<MessageUid> search(MailboxSession session, Mailbox mailbox, SearchQuery searchQuery) throws MailboxException {
- Preconditions.checkArgument(session != null, "'session' is mandatory");
-
- return searchMultimap(ImmutableList.of(mailbox.getMailboxId()), searchQuery)
- .stream()
- .map(SearchResult::getMessageUid)
- .iterator();
- }
-
- @Override
- public List<MessageId> search(MailboxSession session, Collection<MailboxId> mailboxIds, SearchQuery searchQuery, long limit) throws MailboxException {
- Preconditions.checkArgument(session != null, "'session' is mandatory");
- if (mailboxIds.isEmpty()) {
- return ImmutableList.of();
- }
-
- return searchMultimap(mailboxIds, searchQuery)
- .stream()
- .map(searchResult -> searchResult.getMessageId().get())
- .filter(SearchUtil.distinct())
- .limit(Long.valueOf(limit).intValue())
- .collect(Guavate.toImmutableList());
- }
-
- private List<SearchResult> searchMultimap(Collection<MailboxId> mailboxIds, SearchQuery searchQuery) throws MailboxException {
- ImmutableList.Builder<SearchResult> results = ImmutableList.builder();
- IndexSearcher searcher = null;
-
- Query inMailboxes = buildQueryFromMailboxes(mailboxIds);
-
- try {
- searcher = new IndexSearcher(IndexReader.open(writer, true));
- BooleanQuery query = new BooleanQuery();
- query.add(inMailboxes, BooleanClause.Occur.MUST);
- // Not return flags documents
- query.add(new PrefixQuery(new Term(FLAGS_FIELD, "")), BooleanClause.Occur.MUST_NOT);
-
- List<Criterion> crits = searchQuery.getCriterias();
- for (Criterion crit : crits) {
- query.add(createQuery(crit, inMailboxes, searchQuery.getRecentMessageUids()), BooleanClause.Occur.MUST);
- }
-
- // query for all the documents sorted as specified in the SearchQuery
- TopDocs docs = searcher.search(query, null, maxQueryResults, createSort(searchQuery.getSorts()));
- ScoreDoc[] sDocs = docs.scoreDocs;
- for (ScoreDoc sDoc : sDocs) {
- Document doc = searcher.doc(sDoc.doc);
- MessageUid uid = MessageUid.of(Long.valueOf(doc.get(UID_FIELD)));
- MailboxId mailboxId = mailboxIdFactory.fromString(doc.get(MAILBOX_ID_FIELD));
- Optional<MessageId> messageId = toMessageId(Optional.ofNullable(doc.get(MESSAGE_ID_FIELD)));
- results.add(new SearchResult(messageId, mailboxId, uid));
- }
- } catch (IOException e) {
- throw new MailboxException("Unable to search the mailbox", e);
- } finally {
- if (searcher != null) {
- try {
- searcher.close();
- } catch (IOException e) {
- // ignore on close
- }
- }
- }
- return results.build();
- }
-
- private Optional<MessageId> toMessageId(Optional<String> messageIdField) {
- if (messageIdField.isPresent()) {
- return Optional.of(messageIdFactory.fromString(messageIdField.get()));
- }
- return Optional.empty();
- }
-
- private Query buildQueryFromMailboxes(Collection<MailboxId> mailboxIds) {
- BooleanQuery query = new BooleanQuery();
- for (MailboxId id: mailboxIds) {
- String idAsString = id.serialize();
- query.add(new TermQuery(new Term(MAILBOX_ID_FIELD, idAsString)), BooleanClause.Occur.SHOULD);
- }
- return query;
- }
-
-
- /**
- * Create a new {@link Document} for the given {@link MailboxMessage}. This Document does not contain any flags data. The {@link Flags} are stored in a seperate Document.
- *
- * See {@link #createFlagsDocument(MailboxMessage)}
- *
- * @param membership
- * @return document
- */
- private Document createMessageDocument(final MailboxSession session, final MailboxMessage membership) throws MailboxException {
- final Document doc = new Document();
- // TODO: Better handling
- doc.add(new Field(USERS, session.getUser().getUserName().toUpperCase(Locale.US), Store.YES, Index.NOT_ANALYZED));
- doc.add(new Field(MAILBOX_ID_FIELD, membership.getMailboxId().serialize().toUpperCase(Locale.US), Store.YES, Index.NOT_ANALYZED));
- doc.add(new NumericField(UID_FIELD,Store.YES, true).setLongValue(membership.getUid().asLong()));
- doc.add(new Field(HAS_ATTACHMENT_FIELD, Boolean.toString(hasAttachment(membership)), Store.YES, Index.NOT_ANALYZED));
- doc.add(new Field(MESSAGE_ID_FIELD, SearchUtil.getSerializedMessageIdIfSupportedByUnderlyingStorageOrNull(membership), Store.YES, Index.NOT_ANALYZED));
-
- // create an unqiue key for the document which can be used later on updates to find the document
- doc.add(new Field(ID_FIELD, membership.getMailboxId().serialize().toUpperCase(Locale.US) + "-" + Long.toString(membership.getUid().asLong()), Store.YES, Index.NOT_ANALYZED));
-
- doc.add(new Field(INTERNAL_DATE_FIELD_YEAR_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.YEAR), Store.NO, Index.NOT_ANALYZED));
- doc.add(new Field(INTERNAL_DATE_FIELD_MONTH_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.MONTH), Store.NO, Index.NOT_ANALYZED));
- doc.add(new Field(INTERNAL_DATE_FIELD_DAY_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.DAY), Store.NO, Index.NOT_ANALYZED));
- doc.add(new Field(INTERNAL_DATE_FIELD_HOUR_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.HOUR), Store.NO, Index.NOT_ANALYZED));
- doc.add(new Field(INTERNAL_DATE_FIELD_MINUTE_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.MINUTE), Store.NO, Index.NOT_ANALYZED));
- doc.add(new Field(INTERNAL_DATE_FIELD_SECOND_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.SECOND), Store.NO, Index.NOT_ANALYZED));
- doc.add(new Field(INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.MILLISECOND), Store.NO, Index.NOT_ANALYZED));
-
- doc.add(new NumericField(SIZE_FIELD,Store.YES, true).setLongValue(membership.getFullContentOctets()));
-
- // content handler which will index the headers and the body of the message
- SimpleContentHandler handler = new SimpleContentHandler() {
-
-
- @Override
- public void headers(Header header) {
-
- Date sentDate = null;
- String firstFromMailbox = "";
- String firstToMailbox = "";
- String firstCcMailbox = "";
- String firstFromDisplay = "";
- String firstToDisplay = "";
-
- Iterator<org.apache.james.mime4j.stream.Field> fields = header.iterator();
- while (fields.hasNext()) {
- org.apache.james.mime4j.stream.Field f = fields.next();
- String headerName = f.getName().toUpperCase(Locale.US);
- String headerValue = f.getBody().toUpperCase(Locale.US);
- String fullValue = f.toString().toUpperCase(Locale.US);
- doc.add(new Field(HEADERS_FIELD, fullValue, Store.NO, Index.ANALYZED));
- doc.add(new Field(PREFIX_HEADER_FIELD + headerName, headerValue, Store.NO, Index.ANALYZED));
-
- if (f instanceof DateTimeField) {
- // We need to make sure we convert it to GMT
- final StringReader reader = new StringReader(f.getBody());
- try {
- DateTime dateTime = new DateTimeParser(reader).parseAll();
- Calendar cal = getGMT();
- cal.set(dateTime.getYear(), dateTime.getMonth() - 1, dateTime.getDay(), dateTime.getHour(), dateTime.getMinute(), dateTime.getSecond());
- sentDate = cal.getTime();
-
- } catch (org.apache.james.mime4j.field.datetime.parser.ParseException e) {
- LOGGER.debug("Unable to parse Date header for proper indexing", e);
- // This should never happen anyway fallback to the already parsed field
- sentDate = ((DateTimeField) f).getDate();
- }
- if (sentDate == null) {
- sentDate = membership.getInternalDate();
- }
-
- }
- String field = null;
- if ("To".equalsIgnoreCase(headerName)) {
- field = TO_FIELD;
- } else if ("From".equalsIgnoreCase(headerName)) {
- field = FROM_FIELD;
- } else if ("Cc".equalsIgnoreCase(headerName)) {
- field = CC_FIELD;
- } else if ("Bcc".equalsIgnoreCase(headerName)) {
- field = BCC_FIELD;
- }
-
-
- // Check if we can index the the address in the right manner
- if (field != null) {
- // not sure if we really should reparse it. It maybe be better to check just for the right type.
- // But this impl was easier in the first place
- AddressList aList = LenientAddressParser.DEFAULT.parseAddressList(MimeUtil.unfold(f.getBody()));
- for (int i = 0; i < aList.size(); i++) {
- Address address = aList.get(i);
- if (address instanceof org.apache.james.mime4j.dom.address.Mailbox) {
- org.apache.james.mime4j.dom.address.Mailbox mailbox = (org.apache.james.mime4j.dom.address.Mailbox) address;
- String value = AddressFormatter.DEFAULT.encode(mailbox).toUpperCase(Locale.US);
- doc.add(new Field(field, value, Store.NO, Index.ANALYZED));
- if (i == 0) {
- String mailboxAddress = SearchUtil.getMailboxAddress(mailbox);
- String mailboxDisplay = SearchUtil.getDisplayAddress(mailbox);
-
- if ("To".equalsIgnoreCase(headerName)) {
- firstToMailbox = mailboxAddress;
- firstToDisplay = mailboxDisplay;
- } else if ("From".equalsIgnoreCase(headerName)) {
- firstFromMailbox = mailboxAddress;
- firstFromDisplay = mailboxDisplay;
-
- } else if ("Cc".equalsIgnoreCase(headerName)) {
- firstCcMailbox = mailboxAddress;
- }
-
- }
- } else if (address instanceof Group) {
- MailboxList mList = ((Group) address).getMailboxes();
- for (int a = 0; a < mList.size(); a++) {
- org.apache.james.mime4j.dom.address.Mailbox mailbox = mList.get(a);
- String value = AddressFormatter.DEFAULT.encode(mailbox).toUpperCase(Locale.US);
- doc.add(new Field(field, value, Store.NO, Index.ANALYZED));
-
- if (i == 0 && a == 0) {
- String mailboxAddress = SearchUtil.getMailboxAddress(mailbox);
- String mailboxDisplay = SearchUtil.getDisplayAddress(mailbox);
-
- if ("To".equalsIgnoreCase(headerName)) {
- firstToMailbox = mailboxAddress;
- firstToDisplay = mailboxDisplay;
- } else if ("From".equalsIgnoreCase(headerName)) {
- firstFromMailbox = mailboxAddress;
- firstFromDisplay = mailboxDisplay;
-
- } else if ("Cc".equalsIgnoreCase(headerName)) {
- firstCcMailbox = mailboxAddress;
- }
- }
- }
- }
- }
-
-
- doc.add(new Field(field, headerValue, Store.NO, Index.ANALYZED));
-
- } else if (headerName.equalsIgnoreCase("Subject")) {
- doc.add(new Field(BASE_SUBJECT_FIELD, SearchUtil.getBaseSubject(headerValue), Store.YES, Index.NOT_ANALYZED));
- }
- }
- if (sentDate == null) {
- sentDate = membership.getInternalDate();
- } else {
-
- doc.add(new Field(SENT_DATE_FIELD_YEAR_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.YEAR), Store.NO, Index.NOT_ANALYZED));
- doc.add(new Field(SENT_DATE_FIELD_MONTH_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.MONTH), Store.NO, Index.NOT_ANALYZED));
- doc.add(new Field(SENT_DATE_FIELD_DAY_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.DAY), Store.NO, Index.NOT_ANALYZED));
- doc.add(new Field(SENT_DATE_FIELD_HOUR_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.HOUR), Store.NO, Index.NOT_ANALYZED));
- doc.add(new Field(SENT_DATE_FIELD_MINUTE_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.MINUTE), Store.NO, Index.NOT_ANALYZED));
- doc.add(new Field(SENT_DATE_FIELD_SECOND_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.SECOND), Store.NO, Index.NOT_ANALYZED));
- doc.add(new Field(SENT_DATE_FIELD_MILLISECOND_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.MILLISECOND), Store.NO, Index.NOT_ANALYZED));
-
- }
- doc.add(new Field(SENT_DATE_SORT_FIELD_MILLISECOND_RESOLUTION,DateTools.dateToString(sentDate, DateTools.Resolution.MILLISECOND), Store.NO, Index.NOT_ANALYZED));
-
- doc.add(new Field(FIRST_FROM_MAILBOX_NAME_FIELD, firstFromMailbox, Store.YES, Index.NOT_ANALYZED));
- doc.add(new Field(FIRST_TO_MAILBOX_NAME_FIELD, firstToMailbox, Store.YES, Index.NOT_ANALYZED));
- doc.add(new Field(FIRST_CC_MAILBOX_NAME_FIELD, firstCcMailbox, Store.YES, Index.NOT_ANALYZED));
- doc.add(new Field(FIRST_FROM_MAILBOX_DISPLAY_FIELD, firstFromDisplay, Store.YES, Index.NOT_ANALYZED));
- doc.add(new Field(FIRST_TO_MAILBOX_DISPLAY_FIELD, firstToDisplay, Store.YES, Index.NOT_ANALYZED));
-
- }
-
- @Override
- public void body(BodyDescriptor desc, InputStream in) throws MimeException, IOException {
- String mediaType = desc.getMediaType();
- if (MEDIA_TYPE_TEXT.equalsIgnoreCase(mediaType) || MEDIA_TYPE_MESSAGE.equalsIgnoreCase(mediaType)) {
- String cset = desc.getCharset();
- if (cset == null) {
- cset = DEFAULT_ENCODING;
- }
- Charset charset;
- try {
- charset = Charset.forName(cset);
- } catch (Exception e) {
- // Invalid charset found so fallback toe the DEFAULT_ENCODING
- charset = Charset.forName(DEFAULT_ENCODING);
- }
-
- // Read the content one line after the other and add it to the document
- BufferedReader bodyReader = new BufferedReader(new InputStreamReader(in, charset));
- String line = null;
- while ((line = bodyReader.readLine()) != null) {
- doc.add(new Field(BODY_FIELD, line.toUpperCase(Locale.US),Store.NO, Index.ANALYZED));
- }
-
- }
- }
-
- };
- //config.setStrictParsing(false);
- MimeStreamParser parser = new MimeStreamParser(MimeConfig.PERMISSIVE);
- parser.setContentDecoding(true);
- parser.setContentHandler(handler);
-
- try {
- // parse the message to index headers and body
- parser.parse(membership.getFullContent());
- } catch (MimeException | IOException e) {
- // This should never happen as it was parsed before too without problems.
- throw new MailboxException("Unable to index content of message", e);
- }
-
-
- return doc;
- }
-
- private static boolean hasAttachment(MailboxMessage membership) {
- return membership.getProperties().stream()
- .anyMatch(PropertyBuilder.isHasAttachmentProperty());
- }
-
- private String toSentDateField(DateResolution res) {
- switch (res) {
- case Year:
- return SENT_DATE_FIELD_YEAR_RESOLUTION;
- case Month:
- return SENT_DATE_FIELD_MONTH_RESOLUTION;
- case Day:
- return SENT_DATE_FIELD_DAY_RESOLUTION;
- case Hour:
- return SENT_DATE_FIELD_HOUR_RESOLUTION;
- case Minute:
- return SENT_DATE_FIELD_MINUTE_RESOLUTION;
- case Second:
- return SENT_DATE_FIELD_SECOND_RESOLUTION;
- default:
- return SENT_DATE_FIELD_MILLISECOND_RESOLUTION;
- }
- }
-
- private static Calendar getGMT() {
- return Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.US);
- }
-
- private String toInteralDateField(DateResolution res) {
- switch (res) {
- case Year:
- return INTERNAL_DATE_FIELD_YEAR_RESOLUTION;
- case Month:
- return INTERNAL_DATE_FIELD_MONTH_RESOLUTION;
- case Day:
- return INTERNAL_DATE_FIELD_DAY_RESOLUTION;
- case Hour:
- return INTERNAL_DATE_FIELD_HOUR_RESOLUTION;
- case Minute:
- return INTERNAL_DATE_FIELD_MINUTE_RESOLUTION;
- case Second:
- return INTERNAL_DATE_FIELD_SECOND_RESOLUTION;
- default:
- return INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION;
- }
- }
-
- /**
- * Return a {@link Query} which is build based on the given {@link SearchQuery.InternalDateCriterion}
- *
- * @param crit
- * @return query
- * @throws UnsupportedSearchException
- */
- private Query createInternalDateQuery(SearchQuery.InternalDateCriterion crit) throws UnsupportedSearchException {
- DateOperator dop = crit.getOperator();
- DateResolution res = dop.getDateResultion();
- String field = toInteralDateField(res);
- return createQuery(field, dop);
- }
-
- /**
- * Return a {@link Query} which is build based on the given {@link SearchQuery.SizeCriterion}
- *
- * @param crit
- * @return query
- * @throws UnsupportedSearchException
- */
- private Query createSizeQuery(SearchQuery.SizeCriterion crit) throws UnsupportedSearchException {
- NumericOperator op = crit.getOperator();
- switch (op.getType()) {
- case EQUALS:
- return NumericRangeQuery.newLongRange(SIZE_FIELD, op.getValue(), op.getValue(), true, true);
- case GREATER_THAN:
- return NumericRangeQuery.newLongRange(SIZE_FIELD, op.getValue(), Long.MAX_VALUE, false, true);
- case LESS_THAN:
- return NumericRangeQuery.newLongRange(SIZE_FIELD, Long.MIN_VALUE, op.getValue(), true, false);
- default:
- throw new UnsupportedSearchException();
- }
- }
-
- /**
- * This method will return the right {@link Query} depending if {@link #suffixMatch} is enabled
- *
- * @param fieldName
- * @param value
- * @return query
- */
- private Query createTermQuery(String fieldName, String value) {
- if (suffixMatch) {
- return new WildcardQuery(new Term(fieldName, "*" + value + "*"));
- } else {
- return new PrefixQuery(new Term(fieldName, value));
- }
- }
-
- /**
- * Return a {@link Query} which is build based on the given {@link SearchQuery.HeaderCriterion}
- *
- * @param crit
- * @return query
- * @throws UnsupportedSearchException
- */
- private Query createHeaderQuery(SearchQuery.HeaderCriterion crit) throws UnsupportedSearchException {
- HeaderOperator op = crit.getOperator();
- String name = crit.getHeaderName().toUpperCase(Locale.US);
- String fieldName = PREFIX_HEADER_FIELD + name;
- if (op instanceof SearchQuery.ContainsOperator) {
- ContainsOperator cop = (ContainsOperator) op;
- return createTermQuery(fieldName, cop.getValue().toUpperCase(Locale.US));
- } else if (op instanceof SearchQuery.ExistsOperator) {
- return new PrefixQuery(new Term(fieldName, ""));
- } else if (op instanceof SearchQuery.DateOperator) {
- DateOperator dop = (DateOperator) op;
- String field = toSentDateField(dop.getDateResultion());
- return createQuery(field, dop);
- } else if (op instanceof SearchQuery.AddressOperator) {
- String field = name.toLowerCase(Locale.US);
- return createTermQuery(field, ((SearchQuery.AddressOperator) op).getAddress().toUpperCase(Locale.US));
- } else {
- // Operator not supported
- throw new UnsupportedSearchException();
- }
- }
-
-
- private Query createQuery(String field, DateOperator dop) throws UnsupportedSearchException {
- Date date = dop.getDate();
- DateResolution res = dop.getDateResultion();
- DateTools.Resolution dRes = toResolution(res);
- String value = DateTools.dateToString(date, dRes);
- switch (dop.getType()) {
- case ON:
- return new TermQuery(new Term(field, value));
- case BEFORE:
- return new TermRangeQuery(field, DateTools.dateToString(MIN_DATE, dRes), value, true, false);
- case AFTER:
- return new TermRangeQuery(field, value, DateTools.dateToString(MAX_DATE, dRes), false, true);
- default:
- throw new UnsupportedSearchException();
- }
- }
-
- private DateTools.Resolution toResolution(DateResolution res) {
- switch (res) {
- case Year:
- return DateTools.Resolution.YEAR;
- case Month:
- return DateTools.Resolution.MONTH;
- case Day:
- return DateTools.Resolution.DAY;
- case Hour:
- return DateTools.Resolution.HOUR;
- case Minute:
- return DateTools.Resolution.MINUTE;
- case Second:
- return DateTools.Resolution.SECOND;
- default:
- return DateTools.Resolution.MILLISECOND;
- }
- }
-
- /**
- * Return a {@link Query} which is build based on the given {@link SearchQuery.UidCriterion}
- */
- private Query createUidQuery(SearchQuery.UidCriterion crit) throws UnsupportedSearchException {
- UidRange[] ranges = crit.getOperator().getRange();
- if (ranges.length == 1) {
- UidRange range = ranges[0];
- return NumericRangeQuery.newLongRange(UID_FIELD, range.getLowValue().asLong(), range.getHighValue().asLong(), true, true);
- } else {
- BooleanQuery rangesQuery = new BooleanQuery();
- for (UidRange range : ranges) {
- rangesQuery.add(NumericRangeQuery.newLongRange(UID_FIELD, range.getLowValue().asLong(), range.getHighValue().asLong(), true, true), BooleanClause.Occur.SHOULD);
- }
- return rangesQuery;
- }
- }
-
-
- /**
- * Return a {@link Query} which is build based on the given {@link SearchQuery.UidCriterion}
- *
- * @param crit
- * @return query
- * @throws UnsupportedSearchException
- */
- private Query createModSeqQuery(SearchQuery.ModSeqCriterion crit) throws UnsupportedSearchException {
- NumericOperator op = crit.getOperator();
- switch (op.getType()) {
- case EQUALS:
- return NumericRangeQuery.newLongRange(MODSEQ_FIELD, op.getValue(), op.getValue(), true, true);
- case GREATER_THAN:
- return NumericRangeQuery.newLongRange(MODSEQ_FIELD, op.getValue(), Long.MAX_VALUE, false, true);
- case LESS_THAN:
- return NumericRangeQuery.newLongRange(MODSEQ_FIELD, Long.MIN_VALUE, op.getValue(), true, false);
- default:
- throw new UnsupportedSearchException();
- }
- }
-
- private Query createAttachmentQuery(boolean isSet) throws MailboxException {
- return new TermQuery(new Term(HAS_ATTACHMENT_FIELD, Boolean.toString(isSet)));
- }
-
- /**
- * Return a {@link Query} which is build based on the given {@link SearchQuery.FlagCriterion}. This is kind of a hack
- * as it will do a search for the flags in this method and
- *
- * @return query
- * @throws MailboxException
- */
- private Query createFlagQuery(String flag, boolean isSet, Query inMailboxes, Collection<MessageUid> recentUids) throws MailboxException {
- BooleanQuery query = new BooleanQuery();
-
- if (isSet) {
- query.add(new TermQuery(new Term(FLAGS_FIELD, flag)), BooleanClause.Occur.MUST);
- } else {
- // lucene does not support simple NOT queries so we do some nasty hack here
- BooleanQuery bQuery = new BooleanQuery();
- bQuery.add(new PrefixQuery(new Term(FLAGS_FIELD, "")), BooleanClause.Occur.MUST);
- bQuery.add(new TermQuery(new Term(FLAGS_FIELD, flag)),BooleanClause.Occur.MUST_NOT);
-
- query.add(bQuery, BooleanClause.Occur.MUST);
- }
- query.add(inMailboxes, BooleanClause.Occur.MUST);
-
-
- IndexSearcher searcher = null;
-
- try {
- Set<MessageUid> uids = new HashSet<>();
- searcher = new IndexSearcher(IndexReader.open(writer, true));
-
- // query for all the documents sorted by uid
- TopDocs docs = searcher.search(query, null, maxQueryResults, new Sort(UID_SORT));
- ScoreDoc[] sDocs = docs.scoreDocs;
- for (ScoreDoc sDoc : sDocs) {
- MessageUid uid = MessageUid.of(Long.valueOf(searcher.doc(sDoc.doc).get(UID_FIELD)));
- uids.add(uid);
- }
-
- // add or remove recent uids
- if (flag.equalsIgnoreCase("\\RECENT")) {
- if (isSet) {
- uids.addAll(recentUids);
- } else {
- uids.removeAll(recentUids);
- }
- }
-
- List<MessageRange> ranges = MessageRange.toRanges(new ArrayList<>(uids));
- UidRange[] nRanges = new UidRange[ranges.size()];
- for (int i = 0; i < ranges.size(); i++) {
- MessageRange range = ranges.get(i);
- nRanges[i] = new UidRange(range.getUidFrom(), range.getUidTo());
- }
- return createUidQuery((UidCriterion) SearchQuery.uid(nRanges));
- } catch (IOException e) {
- throw new MailboxException("Unable to search mailbox " + inMailboxes, e);
- } finally {
- if (searcher != null) {
- try {
- searcher.close();
- } catch (IOException e) {
- // ignore on close
- }
- }
- }
- }
-
- private Sort createSort(List<SearchQuery.Sort> sorts) {
- List<SortField> fields = new ArrayList<>();
-
- for (SearchQuery.Sort sort : sorts) {
- boolean reverse = sort.isReverse();
- SortField sortField = createSortField(sort, reverse);
- if (sortField != null) {
-
- fields.add(sortField);
-
- // Add the uid sort as tie-breaker
- if (sortField == SENT_DATE_SORT) {
- fields.add(UID_SORT);
- } else if (sortField == SENT_DATE_SORT_REVERSE) {
- fields.add(UID_SORT_REVERSE);
- }
- }
- }
- // add the uid sorting as last so if no other sorting was able to do the job it will get sorted by the uid
- fields.add(UID_SORT);
- Sort sort = new Sort();
- sort.setSort(fields.toArray(new SortField[0]));
- return sort;
- }
-
- private SortField createSortField(SearchQuery.Sort s, boolean reverse) {
- switch (s.getSortClause()) {
- case Arrival:
- if (reverse) {
- return ARRIVAL_MAILBOX_SORT_REVERSE;
- }
- return ARRIVAL_MAILBOX_SORT;
- case SentDate:
- if (reverse) {
- return SENT_DATE_SORT_REVERSE;
- }
- return SENT_DATE_SORT;
- case MailboxCc:
- if (reverse) {
- return FIRST_CC_MAILBOX_SORT_REVERSE;
- }
- return FIRST_CC_MAILBOX_SORT;
- case MailboxFrom:
- if (reverse) {
- return FIRST_FROM_MAILBOX_SORT_REVERSE;
- }
- return FIRST_FROM_MAILBOX_SORT;
- case Size:
- if (reverse) {
- return SIZE_SORT_REVERSE;
- }
- return SIZE_SORT;
- case BaseSubject:
- if (reverse) {
- return BASE_SUBJECT_SORT_REVERSE;
- }
- return BASE_SUBJECT_SORT;
- case MailboxTo:
- if (reverse) {
- return FIRST_TO_MAILBOX_SORT_REVERSE;
- }
- return FIRST_TO_MAILBOX_SORT;
- case Uid:
- if (reverse) {
- return UID_SORT_REVERSE;
- }
- return UID_SORT;
- case DisplayFrom:
- if (reverse) {
- return FIRST_FROM_MAILBOX_DISPLAY_SORT_REVERSE;
- }
- return FIRST_FROM_MAILBOX_DISPLAY_SORT;
- case DisplayTo:
- if (reverse) {
- return FIRST_TO_MAILBOX_DISPLAY_SORT_REVERSE;
- }
- return FIRST_TO_MAILBOX_DISPLAY_SORT;
- default:
- return null;
- }
- }
-
- /**
- * Convert the given {@link Flag} to a String
- *
- * @param flag
- * @return flagString
- */
- private String toString(Flag flag) {
- if (Flag.ANSWERED.equals(flag)) {
- return "\\ANSWERED";
- } else if (Flag.DELETED.equals(flag)) {
- return "\\DELETED";
- } else if (Flag.DRAFT.equals(flag)) {
- return "\\DRAFT";
- } else if (Flag.FLAGGED.equals(flag)) {
- return "\\FLAGGED";
- } else if (Flag.RECENT.equals(flag)) {
- return "\\RECENT";
- } else if (Flag.SEEN.equals(flag)) {
- return "\\FLAG";
- } else {
- return flag.toString();
- }
- }
-
- /**
- * Return a {@link Query} which is build based on the given {@link SearchQuery.TextCriterion}
- *
- * @param crit
- * @return query
- * @throws UnsupportedSearchException
- */
- private Query createTextQuery(SearchQuery.TextCriterion crit) throws UnsupportedSearchException {
- String value = crit.getOperator().getValue().toUpperCase(Locale.US);
- switch (crit.getType()) {
- case BODY:
- return createTermQuery(BODY_FIELD, value);
- case FULL:
- BooleanQuery query = new BooleanQuery();
- query.add(createTermQuery(BODY_FIELD, value), BooleanClause.Occur.SHOULD);
- query.add(createTermQuery(HEADERS_FIELD,value), BooleanClause.Occur.SHOULD);
- return query;
- default:
- throw new UnsupportedSearchException();
- }
- }
-
- /**
- * Return a {@link Query} which is build based on the given {@link SearchQuery.AllCriterion}
- *
- * @param crit
- * @return query
- * @throws UnsupportedSearchException
- */
- private Query createAllQuery(SearchQuery.AllCriterion crit) throws UnsupportedSearchException {
- BooleanQuery query = new BooleanQuery();
-
- query.add(createQuery(MessageRange.all()), BooleanClause.Occur.MUST);
- query.add(new PrefixQuery(new Term(FLAGS_FIELD, "")), BooleanClause.Occur.MUST_NOT);
-
- return query;
- }
-
- /**
- * Return a {@link Query} which is build based on the given {@link SearchQuery.ConjunctionCriterion}
- *
- * @param crit
- * @return query
- * @throws UnsupportedSearchException
- */
- private Query createConjunctionQuery(SearchQuery.ConjunctionCriterion crit, Query inMailboxes, Collection<MessageUid> recentUids) throws UnsupportedSearchException, MailboxException {
- List<Criterion> crits = crit.getCriteria();
- BooleanQuery conQuery = new BooleanQuery();
- switch (crit.getType()) {
- case AND:
- for (Criterion criterion : crits) {
- conQuery.add(createQuery(criterion, inMailboxes, recentUids), BooleanClause.Occur.MUST);
- }
- return conQuery;
- case OR:
- for (Criterion criterion : crits) {
- conQuery.add(createQuery(criterion, inMailboxes, recentUids), BooleanClause.Occur.SHOULD);
- }
- return conQuery;
- case NOR:
- BooleanQuery nor = new BooleanQuery();
- for (Criterion criterion : crits) {
- conQuery.add(createQuery(criterion, inMailboxes, recentUids), BooleanClause.Occur.SHOULD);
- }
- nor.add(inMailboxes, BooleanClause.Occur.MUST);
-
- nor.add(conQuery, BooleanClause.Occur.MUST_NOT);
- return nor;
- default:
- throw new UnsupportedSearchException();
- }
-
- }
-
- /**
- * Return a {@link Query} which is builded based on the given {@link Criterion}
- *
- * @param criterion
- * @return query
- * @throws UnsupportedSearchException
- */
- private Query createQuery(Criterion criterion, Query inMailboxes, Collection<MessageUid> recentUids) throws MailboxException {
- if (criterion instanceof SearchQuery.InternalDateCriterion) {
- SearchQuery.InternalDateCriterion crit = (SearchQuery.InternalDateCriterion) criterion;
- return createInternalDateQuery(crit);
- } else if (criterion instanceof SearchQuery.SizeCriterion) {
- SearchQuery.SizeCriterion crit = (SearchQuery.SizeCriterion) criterion;
- return createSizeQuery(crit);
- } else if (criterion instanceof SearchQuery.HeaderCriterion) {
- HeaderCriterion crit = (HeaderCriterion) criterion;
- return createHeaderQuery(crit);
- } else if (criterion instanceof SearchQuery.UidCriterion) {
- SearchQuery.UidCriterion crit = (SearchQuery.UidCriterion) criterion;
- return createUidQuery(crit);
- } else if (criterion instanceof SearchQuery.FlagCriterion) {
- FlagCriterion crit = (FlagCriterion) criterion;
- return createFlagQuery(toString(crit.getFlag()), crit.getOperator().isSet(), inMailboxes, recentUids);
- } else if (criterion instanceof SearchQuery.AttachmentCriterion) {
- AttachmentCriterion crit = (AttachmentCriterion) criterion;
- return createAttachmentQuery(crit.getOperator().isSet());
- } else if (criterion instanceof SearchQuery.CustomFlagCriterion) {
- CustomFlagCriterion crit = (CustomFlagCriterion) criterion;
- return createFlagQuery(crit.getFlag(), crit.getOperator().isSet(), inMailboxes, recentUids);
- } else if (criterion instanceof SearchQuery.TextCriterion) {
- SearchQuery.TextCriterion crit = (SearchQuery.TextCriterion) criterion;
- return createTextQuery(crit);
- } else if (criterion instanceof SearchQuery.AllCriterion) {
- return createAllQuery((AllCriterion) criterion);
- } else if (criterion instanceof SearchQuery.ConjunctionCriterion) {
- SearchQuery.ConjunctionCriterion crit = (SearchQuery.ConjunctionCriterion) criterion;
- return createConjunctionQuery(crit, inMailboxes, recentUids);
- } else if (criterion instanceof SearchQuery.ModSeqCriterion) {
- return createModSeqQuery((SearchQuery.ModSeqCriterion) criterion);
- } else if (criterion instanceof SearchQuery.MimeMessageIDCriterion) {
- SearchQuery.MimeMessageIDCriterion mimeMessageIDCriterion = (SearchQuery.MimeMessageIDCriterion) criterion;
- return createHeaderQuery(mimeMessageIDCriterion.asHeaderCriterion());
- }
- throw new UnsupportedSearchException();
- }
-
- @Override
- public void add(MailboxSession session, Mailbox mailbox, MailboxMessage membership) throws MailboxException {
- Document doc = createMessageDocument(session, membership);
- Document flagsDoc = createFlagsDocument(membership);
-
- try {
- writer.addDocument(doc);
- writer.addDocument(flagsDoc);
- } catch (IOException e) {
- throw new MailboxException("Unable to add message to index", e);
- }
- }
-
- @Override
- public void update(MailboxSession session, Mailbox mailbox, List<UpdatedFlags> updatedFlagsList) throws MailboxException {
- for (UpdatedFlags updatedFlags : updatedFlagsList) {
- update(mailbox, updatedFlags.getUid(), updatedFlags.getNewFlags());
- }
- }
-
- private void update(Mailbox mailbox, MessageUid uid, Flags f) throws MailboxException {
- IndexSearcher searcher = null;
- try {
- searcher = new IndexSearcher(IndexReader.open(writer, true));
- BooleanQuery query = new BooleanQuery();
- query.add(new TermQuery(new Term(MAILBOX_ID_FIELD, mailbox.getMailboxId().serialize())), BooleanClause.Occur.MUST);
- query.add(createQuery(MessageRange.one(uid)), BooleanClause.Occur.MUST);
- query.add(new PrefixQuery(new Term(FLAGS_FIELD, "")), BooleanClause.Occur.MUST);
-
- TopDocs docs = searcher.search(query, 100000);
- ScoreDoc[] sDocs = docs.scoreDocs;
- for (ScoreDoc sDoc : sDocs) {
- Document doc = searcher.doc(sDoc.doc);
-
- if (doc.getFieldable(FLAGS_FIELD) == null) {
- doc.removeFields(FLAGS_FIELD);
- indexFlags(doc, f);
-
- writer.updateDocument(new Term(ID_FIELD, doc.get(ID_FIELD)), doc);
-
- }
- }
- } catch (IOException e) {
- throw new MailboxException("Unable to add messages in index", e);
-
- } finally {
- try {
- IOUtils.closeWhileHandlingException(searcher);
- } catch (IOException e) {
- //can't happen anyway
- }
- }
-
- }
-
- /**
- * Index the {@link Flags} and add it to the {@link Document}
- */
- private Document createFlagsDocument(MailboxMessage message) {
- Document doc = new Document();
- doc.add(new Field(ID_FIELD, "flags-" + message.getMailboxId().serialize() + "-" + Long.toString(message.getUid().asLong()), Store.YES, Index.NOT_ANALYZED));
- doc.add(new Field(MAILBOX_ID_FIELD, message.getMailboxId().serialize(), Store.YES, Index.NOT_ANALYZED));
- doc.add(new NumericField(UID_FIELD,Store.YES, true).setLongValue(message.getUid().asLong()));
-
- indexFlags(doc, message.createFlags());
- return doc;
- }
-
- /**
- * Add the given {@link Flags} to the {@link Document}
- *
- * @param doc
- * @param f
- */
- private void indexFlags(Document doc, Flags f) {
- List<String> fString = new ArrayList<>();
- Flag[] flags = f.getSystemFlags();
- for (Flag flag : flags) {
- fString.add(toString(flag));
- doc.add(new Field(FLAGS_FIELD, toString(flag), Store.NO, Index.NOT_ANALYZED));
- }
-
- String[] userFlags = f.getUserFlags();
- for (String userFlag : userFlags) {
- doc.add(new Field(FLAGS_FIELD, userFlag, Store.NO, Index.NOT_ANALYZED));
- }
-
- // if no flags are there we just use a empty field
- if (flags.length == 0 && userFlags.length == 0) {
- doc.add(new Field(FLAGS_FIELD, "",Store.NO, Index.NOT_ANALYZED));
- }
-
- }
-
- private Query createQuery(MessageRange range) {
- switch (range.getType()) {
- case ONE:
- return NumericRangeQuery.newLongRange(UID_FIELD,
- range.getUidFrom().asLong(),
- range.getUidTo().asLong(), true, true);
- case FROM:
- return NumericRangeQuery.newLongRange(UID_FIELD,
- range.getUidFrom().asLong(),
- MessageUid.MAX_VALUE.asLong(), true, true);
- default:
- return NumericRangeQuery.newLongRange(UID_FIELD, MessageUid.MIN_VALUE.asLong(), MessageUid.MAX_VALUE.asLong(), true, true);
- }
- }
-
- @Override
- public void delete(MailboxSession session, Mailbox mailbox, List<MessageUid> expungedUids) throws MailboxException {
- Collection<MessageRange> messageRanges = MessageRange.toRanges(expungedUids);
- for (MessageRange messageRange : messageRanges) {
- delete(mailbox, messageRange);
- }
- }
-
- @Override
- public void deleteAll(MailboxSession session, Mailbox mailbox) throws MailboxException {
- delete(mailbox, MessageRange.all());
- }
-
- public void delete(Mailbox mailbox, MessageRange range) throws MailboxException {
- BooleanQuery query = new BooleanQuery();
- query.add(new TermQuery(new Term(MAILBOX_ID_FIELD, mailbox.getMailboxId().serialize())), BooleanClause.Occur.MUST);
- query.add(createQuery(range), BooleanClause.Occur.MUST);
-
- try {
- writer.deleteDocuments(query);
- } catch (IOException e) {
- throw new MailboxException("Unable to delete message from index", e);
- }
- }
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+package org.apache.james.mailbox.lucene.search;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.StringReader;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Collection;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TimeZone;
+
+import javax.inject.Inject;
+import javax.mail.Flags;
+import javax.mail.Flags.Flag;
+
+import org.apache.james.mailbox.MailboxManager;
+import org.apache.james.mailbox.MailboxManager.SearchCapabilities;
+import org.apache.james.mailbox.MailboxSession;
+import org.apache.james.mailbox.MessageUid;
+import org.apache.james.mailbox.exception.MailboxException;
+import org.apache.james.mailbox.exception.UnsupportedSearchException;
+import org.apache.james.mailbox.model.MailboxId;
+import org.apache.james.mailbox.model.MessageId;
+import org.apache.james.mailbox.model.MessageRange;
+import org.apache.james.mailbox.model.SearchQuery;
+import org.apache.james.mailbox.model.SearchQuery.AllCriterion;
+import org.apache.james.mailbox.model.SearchQuery.AttachmentCriterion;
+import org.apache.james.mailbox.model.SearchQuery.ContainsOperator;
+import org.apache.james.mailbox.model.SearchQuery.Criterion;
+import org.apache.james.mailbox.model.SearchQuery.CustomFlagCriterion;
+import org.apache.james.mailbox.model.SearchQuery.DateOperator;
+import org.apache.james.mailbox.model.SearchQuery.DateResolution;
+import org.apache.james.mailbox.model.SearchQuery.FlagCriterion;
+import org.apache.james.mailbox.model.SearchQuery.HeaderCriterion;
+import org.apache.james.mailbox.model.SearchQuery.HeaderOperator;
+import org.apache.james.mailbox.model.SearchQuery.NumericOperator;
+import org.apache.james.mailbox.model.SearchQuery.UidCriterion;
+import org.apache.james.mailbox.model.SearchQuery.UidRange;
+import org.apache.james.mailbox.model.UpdatedFlags;
+import org.apache.james.mailbox.store.mail.MessageMapperFactory;
+import org.apache.james.mailbox.store.mail.model.Mailbox;
+import org.apache.james.mailbox.store.mail.model.MailboxMessage;
+import org.apache.james.mailbox.store.mail.model.impl.PropertyBuilder;
+import org.apache.james.mailbox.store.search.ListeningMessageSearchIndex;
+import org.apache.james.mailbox.store.search.SearchUtil;
+import org.apache.james.mime4j.MimeException;
+import org.apache.james.mime4j.dom.Header;
+import org.apache.james.mime4j.dom.address.Address;
+import org.apache.james.mime4j.dom.address.AddressList;
+import org.apache.james.mime4j.dom.address.Group;
+import org.apache.james.mime4j.dom.address.MailboxList;
+import org.apache.james.mime4j.dom.datetime.DateTime;
+import org.apache.james.mime4j.dom.field.DateTimeField;
+import org.apache.james.mime4j.field.address.AddressFormatter;
+import org.apache.james.mime4j.field.address.LenientAddressParser;
+import org.apache.james.mime4j.field.datetime.parser.DateTimeParser;
+import org.apache.james.mime4j.message.SimpleContentHandler;
+import org.apache.james.mime4j.parser.MimeStreamParser;
+import org.apache.james.mime4j.stream.BodyDescriptor;
+import org.apache.james.mime4j.stream.MimeConfig;
+import org.apache.james.mime4j.util.MimeUtil;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.DateTools;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.NumericField;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.NumericRangeQuery;
+import org.apache.lucene.search.PrefixQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Version;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.github.steveash.guavate.Guavate;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Lucene based {@link ListeningMessageSearchIndex} which offers message searching via a Lucene index
+ *
+ *
+
+ * @param
+ */
+public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
+ private static final Logger LOGGER = LoggerFactory.getLogger(LuceneMessageSearchIndex.class);
+ private static final Date MAX_DATE;
+ private static final Date MIN_DATE;
+
+ static {
+ Calendar cal = Calendar.getInstance();
+ cal.set(9999, 11, 31);
+ MAX_DATE = cal.getTime();
+
+ cal.set(0000, 0, 1);
+ MIN_DATE = cal.getTime();
+ }
+
+ /**
+ * Default max query results
+ */
+ private static final int DEFAULT_MAX_QUERY_RESULTS = 100000;
+
+ /**
+ * {@link Field} which will contain the unique index of the {@link Document}
+ */
+ private static final String ID_FIELD = "id";
+
+
+ /**
+ * {@link Field} which will contain uid of the {@link MailboxMessage}
+ */
+ private static final String UID_FIELD = "uid";
+
+ /**
+ * {@link Field} boolean field that say if the message as an attachment or not
+ */
+ private static final String HAS_ATTACHMENT_FIELD = "hasAttachment";
+
+ /**
+ * {@link Field} which will contain the {@link Flags} of the {@link MailboxMessage}
+ */
+ private static final String FLAGS_FIELD = "flags";
+
+ /**
+ * {@link Field} which will contain the size of the {@link MailboxMessage}
+ */
+ private static final String SIZE_FIELD = "size";
+
+ /**
+ * {@link Field} which will contain the body of the {@link MailboxMessage}
+ */
+ private static final String BODY_FIELD = "body";
+
+
+ /**
+ * Prefix which will be used for each message header to store it also in a seperate {@link Field}
+ */
+ private static final String PREFIX_HEADER_FIELD = "header_";
+
+ /**
+ * {@link Field} which will contain the whole message header of the {@link MailboxMessage}
+ */
+ private static final String HEADERS_FIELD = "headers";
+
+ /**
+ * {@link Field} which will contain the mod-sequence of the message
+ */
+ private static final String MODSEQ_FIELD = "modSeq";
+
+ /**
+ * {@link Field} which will contain the TO-Address of the message
+ */
+ private static final String TO_FIELD = "to";
+
+ private static final String FIRST_TO_MAILBOX_NAME_FIELD = "firstToMailboxName";
+ private static final String FIRST_TO_MAILBOX_DISPLAY_FIELD = "firstToMailboxDisplay";
+
+ /**
+ * {@link Field} which will contain the CC-Address of the message
+ */
+ private static final String CC_FIELD = "cc";
+
+ private static final String FIRST_CC_MAILBOX_NAME_FIELD = "firstCcMailboxName";
+
+
+ /**
+ * {@link Field} which will contain the FROM-Address of the message
+ */
+ private static final String FROM_FIELD = "from";
+
+ private static final String FIRST_FROM_MAILBOX_NAME_FIELD = "firstFromMailboxName";
+ private static final String FIRST_FROM_MAILBOX_DISPLAY_FIELD = "firstFromMailboxDisplay";
+
+ /**
+ * {@link Field} which will contain the BCC-Address of the message
+ */
+ private static final String BCC_FIELD = "bcc";
+
+
+ private static final String BASE_SUBJECT_FIELD = "baseSubject";
+
+ /**
+ * {@link Field} which contain the internalDate of the message with YEAR-Resolution
+ */
+ private static final String INTERNAL_DATE_FIELD_YEAR_RESOLUTION = "internaldateYearResolution";
+
+
+ /**
+ * {@link Field} which contain the internalDate of the message with MONTH-Resolution
+ */
+ private static final String INTERNAL_DATE_FIELD_MONTH_RESOLUTION = "internaldateMonthResolution";
+
+ /**
+ * {@link Field} which contain the internalDate of the message with DAY-Resolution
+ */
+ private static final String INTERNAL_DATE_FIELD_DAY_RESOLUTION = "internaldateDayResolution";
+
+ /**
+ * {@link Field} which contain the internalDate of the message with HOUR-Resolution
+ */
+ private static final String INTERNAL_DATE_FIELD_HOUR_RESOLUTION = "internaldateHourResolution";
+
+ /**
+ * {@link Field} which contain the internalDate of the message with MINUTE-Resolution
+ */
+ private static final String INTERNAL_DATE_FIELD_MINUTE_RESOLUTION = "internaldateMinuteResolution";
+
+ /**
+ * {@link Field} which contain the internalDate of the message with SECOND-Resolution
+ */
+ private static final String INTERNAL_DATE_FIELD_SECOND_RESOLUTION = "internaldateSecondResolution";
+
+
+ /**
+ * {@link Field} which contain the internalDate of the message with MILLISECOND-Resolution
+ */
+ private static final String INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION = "internaldateMillisecondResolution";
+
+ /**
+ * {@link Field} which will contain the id of the {@link Mailbox}
+ */
+ private static final String MAILBOX_ID_FIELD = "mailboxid";
+
+ /**
+ * {@link Field} which will contain the user of the {@link MailboxSession}
+ */
+ private static final String USERS = "userSession";
+ /**
+ * {@link Field} which will contain the id of the {@link MessageId}
+ */
+ private static final String MESSAGE_ID_FIELD = "messageid";
+
+ /**
+ * {@link Field} which contain the Date header of the message with YEAR-Resolution
+ */
+ private static final String SENT_DATE_FIELD_YEAR_RESOLUTION = "sentdateYearResolution";
+
+
+ /**
+ * {@link Field} which contain the Date header of the message with MONTH-Resolution
+ */
+ private static final String SENT_DATE_FIELD_MONTH_RESOLUTION = "sentdateMonthResolution";
+
+ /**
+ * {@link Field} which contain the Date header of the message with DAY-Resolution
+ */
+ private static final String SENT_DATE_FIELD_DAY_RESOLUTION = "sentdateDayResolution";
+
+ /**
+ * {@link Field} which contain the Date header of the message with HOUR-Resolution
+ */
+ private static final String SENT_DATE_FIELD_HOUR_RESOLUTION = "sentdateHourResolution";
+
+ /**
+ * {@link Field} which contain the Date header of the message with MINUTE-Resolution
+ */
+ private static final String SENT_DATE_FIELD_MINUTE_RESOLUTION = "sentdateMinuteResolution";
+
+ /**
+ * {@link Field} which contain the Date header of the message with SECOND-Resolution
+ */
+ private static final String SENT_DATE_FIELD_SECOND_RESOLUTION = "sentdateSecondResolution";
+
+
+ /**
+ * {@link Field} which contain the Date header of the message with MILLISECOND-Resolution
+ */
+ private static final String SENT_DATE_FIELD_MILLISECOND_RESOLUTION = "sentdateMillisecondResolution";
+
+ private static final String SENT_DATE_SORT_FIELD_MILLISECOND_RESOLUTION = "sentdateSort";
+
+ private static final String MEDIA_TYPE_TEXT = "text";
+ private static final String MEDIA_TYPE_MESSAGE = "message";
+ private static final String DEFAULT_ENCODING = "US-ASCII";
+
+ private static final SortField UID_SORT = new SortField(UID_FIELD, SortField.LONG);
+ private static final SortField UID_SORT_REVERSE = new SortField(UID_FIELD, SortField.LONG, true);
+
+ private static final SortField SIZE_SORT = new SortField(SIZE_FIELD, SortField.LONG);
+ private static final SortField SIZE_SORT_REVERSE = new SortField(SIZE_FIELD, SortField.LONG, true);
+
+ private static final SortField FIRST_CC_MAILBOX_SORT = new SortField(FIRST_CC_MAILBOX_NAME_FIELD, SortField.STRING);
+ private static final SortField FIRST_CC_MAILBOX_SORT_REVERSE = new SortField(FIRST_CC_MAILBOX_NAME_FIELD, SortField.STRING, true);
+
+ private static final SortField FIRST_TO_MAILBOX_SORT = new SortField(FIRST_TO_MAILBOX_NAME_FIELD, SortField.STRING);
+ private static final SortField FIRST_TO_MAILBOX_SORT_REVERSE = new SortField(FIRST_TO_MAILBOX_NAME_FIELD, SortField.STRING, true);
+
+ private static final SortField FIRST_FROM_MAILBOX_SORT = new SortField(FIRST_FROM_MAILBOX_NAME_FIELD, SortField.STRING);
+ private static final SortField FIRST_FROM_MAILBOX_SORT_REVERSE = new SortField(FIRST_FROM_MAILBOX_NAME_FIELD, SortField.STRING, true);
+
+
+ private static final SortField ARRIVAL_MAILBOX_SORT = new SortField(INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION, SortField.LONG);
+ private static final SortField ARRIVAL_MAILBOX_SORT_REVERSE = new SortField(INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION, SortField.LONG, true);
+
+ private static final SortField BASE_SUBJECT_SORT = new SortField(BASE_SUBJECT_FIELD, SortField.STRING);
+ private static final SortField BASE_SUBJECT_SORT_REVERSE = new SortField(BASE_SUBJECT_FIELD, SortField.STRING, true);
+
+ private static final SortField SENT_DATE_SORT = new SortField(SENT_DATE_SORT_FIELD_MILLISECOND_RESOLUTION, SortField.LONG);
+ private static final SortField SENT_DATE_SORT_REVERSE = new SortField(SENT_DATE_SORT_FIELD_MILLISECOND_RESOLUTION, SortField.LONG, true);
+
+ private static final SortField FIRST_TO_MAILBOX_DISPLAY_SORT = new SortField(FIRST_TO_MAILBOX_DISPLAY_FIELD, SortField.STRING);
+ private static final SortField FIRST_TO_MAILBOX_DISPLAY_SORT_REVERSE = new SortField(FIRST_TO_MAILBOX_DISPLAY_FIELD, SortField.STRING, true);
+
+ private static final SortField FIRST_FROM_MAILBOX_DISPLAY_SORT = new SortField(FIRST_FROM_MAILBOX_DISPLAY_FIELD, SortField.STRING);
+ private static final SortField FIRST_FROM_MAILBOX_DISPLAY_SORT_REVERSE = new SortField(FIRST_FROM_MAILBOX_DISPLAY_FIELD, SortField.STRING, true);
+
+ private final MailboxId.Factory mailboxIdFactory;
+ private final MessageId.Factory messageIdFactory;
+ private final IndexWriter writer;
+
+ private int maxQueryResults = DEFAULT_MAX_QUERY_RESULTS;
+
+ private boolean suffixMatch = false;
+
+ @Inject
+ public LuceneMessageSearchIndex(
+ MessageMapperFactory factory,
+ MailboxId.Factory mailboxIdFactory,
+ Directory directory,
+ MessageId.Factory messageIdFactory
+ ) throws IOException {
+ this(factory, mailboxIdFactory, directory, false, true, messageIdFactory);
+ }
+
+ public LuceneMessageSearchIndex(
+ MessageMapperFactory factory,
+ MailboxId.Factory mailboxIdFactory,
+ Directory directory,
+ boolean dropIndexOnStart,
+ boolean lenient,
+ MessageId.Factory messageIdFactory
+ ) throws IOException {
+ super(factory);
+ this.mailboxIdFactory = mailboxIdFactory;
+ this.messageIdFactory = messageIdFactory;
+ this.writer = new IndexWriter(directory, createConfig(createAnalyzer(lenient), dropIndexOnStart));
+ }
+
+
+ public LuceneMessageSearchIndex(MessageMapperFactory factory, MailboxId.Factory mailboxIdFactory, MessageId.Factory messageIdFactory, IndexWriter writer) {
+ super(factory);
+ this.mailboxIdFactory = mailboxIdFactory;
+ this.messageIdFactory = messageIdFactory;
+ this.writer = writer;
+ }
+
+ @Override
+ public ListenerType getType() {
+ return ListenerType.EACH_NODE;
+ }
+
+
+ @Override
+ public EnumSet<SearchCapabilities> getSupportedCapabilities(EnumSet<MailboxManager.MessageCapabilities> messageCapabilities) {
+ return EnumSet.of(SearchCapabilities.MultimailboxSearch);
+
+ }
+
+ /**
+ * Set the max count of results which will get returned from a query. The default is {@link #DEFAULT_MAX_QUERY_RESULTS}
+ *
+ * @param maxQueryResults
+ */
+ public void setMaxQueryResults(int maxQueryResults) {
+ this.maxQueryResults = maxQueryResults;
+ }
+
+ protected IndexWriterConfig createConfig(Analyzer analyzer, boolean dropIndexOnStart) {
+ IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_31, analyzer);
+ if (dropIndexOnStart) {
+ config.setOpenMode(OpenMode.CREATE);
+ } else {
+ config.setOpenMode(OpenMode.CREATE_OR_APPEND);
+ }
+ return config;
+ }
+
+ /**
+ * Create a {@link Analyzer} which is used to index the {@link MailboxMessage}'s
+ *
+ * @param lenient
+ *
+ * @return analyzer
+ */
+ protected Analyzer createAnalyzer(boolean lenient) {
+ if (lenient) {
+ return new LenientImapSearchAnalyzer();
+ } else {
+ return new StrictImapSearchAnalyzer();
+ }
+ }
+
+ /**
+ * If set to true this implementation will use {@link WildcardQuery} to match suffix and prefix. This is what RFC3501 expects but is often not what the user does.
+ * It also slow things a lot if you have complex queries which use many "TEXT" arguments. If you want the implementation to behave strict like RFC3501 says, you should
+ * set this to true.
+ *
+ * The default is false for performance reasons
+ *
+ *
+ * @param suffixMatch
+ */
+ public void setEnableSuffixMatch(boolean suffixMatch) {
+ this.suffixMatch = suffixMatch;
+ }
+
+
+
+ @Override
+ public Iterator<MessageUid> search(MailboxSession session, Mailbox mailbox, SearchQuery searchQuery) throws MailboxException {
+ Preconditions.checkArgument(session != null, "'session' is mandatory");
+
+ return searchMultimap(ImmutableList.of(mailbox.getMailboxId()), searchQuery)
+ .stream()
+ .map(SearchResult::getMessageUid)
+ .iterator();
+ }
+
+ @Override
+ public List<MessageId> search(MailboxSession session, Collection<MailboxId> mailboxIds, SearchQuery searchQuery, long limit) throws MailboxException {
+ Preconditions.checkArgument(session != null, "'session' is mandatory");
+ if (mailboxIds.isEmpty()) {
+ return ImmutableList.of();
+ }
+
+ return searchMultimap(mailboxIds, searchQuery)
+ .stream()
+ .map(searchResult -> searchResult.getMessageId().get())
+ .filter(SearchUtil.distinct())
+ .limit(Long.valueOf(limit).intValue())
+ .collect(Guavate.toImmutableList());
+ }
+
+ private List<SearchResult> searchMultimap(Collection<MailboxId> mailboxIds, SearchQuery searchQuery) throws MailboxException {
+ ImmutableList.Builder<SearchResult> results = ImmutableList.builder();
+
+ Query inMailboxes = buildQueryFromMailboxes(mailboxIds);
+
+ try (IndexSearcher searcher = new IndexSearcher(IndexReader.open(writer, true))) {
+ BooleanQuery query = new BooleanQuery();
+ query.add(inMailboxes, BooleanClause.Occur.MUST);
+ // Not return flags documents
+ query.add(new PrefixQuery(new Term(FLAGS_FIELD, "")), BooleanClause.Occur.MUST_NOT);
+
+ List<Criterion> crits = searchQuery.getCriterias();
+ for (Criterion crit : crits) {
+ query.add(createQuery(crit, inMailboxes, searchQuery.getRecentMessageUids()), BooleanClause.Occur.MUST);
+ }
+
+ // query for all the documents sorted as specified in the SearchQuery
+ TopDocs docs = searcher.search(query, null, maxQueryResults, createSort(searchQuery.getSorts()));
+ ScoreDoc[] sDocs = docs.scoreDocs;
+ for (ScoreDoc sDoc : sDocs) {
+ Document doc = searcher.doc(sDoc.doc);
+ MessageUid uid = MessageUid.of(Long.valueOf(doc.get(UID_FIELD)));
+ MailboxId mailboxId = mailboxIdFactory.fromString(doc.get(MAILBOX_ID_FIELD));
+ Optional<MessageId> messageId = toMessageId(Optional.ofNullable(doc.get(MESSAGE_ID_FIELD)));
+ results.add(new SearchResult(messageId, mailboxId, uid));
+ }
+ } catch (IOException e) {
+ throw new MailboxException("Unable to search the mailbox", e);
+ }
+ return results.build();
+ }
+
+ private Optional<MessageId> toMessageId(Optional<String> messageIdField) {
+ if (messageIdField.isPresent()) {
+ return Optional.of(messageIdFactory.fromString(messageIdField.get()));
+ }
+ return Optional.empty();
+ }
+
+ private Query buildQueryFromMailboxes(Collection<MailboxId> mailboxIds) {
+ BooleanQuery query = new BooleanQuery();
+ for (MailboxId id: mailboxIds) {
+ String idAsString = id.serialize();
+ query.add(new TermQuery(new Term(MAILBOX_ID_FIELD, idAsString)), BooleanClause.Occur.SHOULD);
+ }
+ return query;
+ }
+
+
+ /**
+ * Create a new {@link Document} for the given {@link MailboxMessage}. This Document does not contain any flags data. The {@link Flags} are stored in a seperate Document.
+ *
+ * See {@link #createFlagsDocument(MailboxMessage)}
+ *
+ * @param membership
+ * @return document
+ */
+ private Document createMessageDocument(final MailboxSession session, final MailboxMessage membership) throws MailboxException {
+ final Document doc = new Document();
+ // TODO: Better handling
+ doc.add(new Field(USERS, session.getUser().getUserName().toUpperCase(Locale.US), Store.YES, Index.NOT_ANALYZED));
+ doc.add(new Field(MAILBOX_ID_FIELD, membership.getMailboxId().serialize().toUpperCase(Locale.US), Store.YES, Index.NOT_ANALYZED));
+ doc.add(new NumericField(UID_FIELD,Store.YES, true).setLongVal
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org