You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by no...@apache.org on 2011/06/01 18:49:05 UTC
svn commit: r1130220 - in /james/mailbox/trunk:
jcr/src/main/java/org/apache/james/mailbox/jcr/
jpa/src/main/java/org/apache/james/mailbox/jpa/
maildir/src/main/java/org/apache/james/mailbox/maildir/
maildir/src/main/java/org/apache/james/mailbox/maild...
Author: norman
Date: Wed Jun 1 16:49:05 2011
New Revision: 1130220
URL: http://svn.apache.org/viewvc?rev=1130220&view=rev
Log:
LuceneMessageSearchIndex now works... See MAILBOX-10
Added:
james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/LazyMessageSearchIndex.java
james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LenientImapSearchAnalyzer.java
james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/StrictImapSearchAnalyzer.java
- copied, changed from r1130081, james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/ImapSearchAnalyzer.java
james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/StrictLuceneMessageSearchIndexText.java
Removed:
james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/ImapSearchAnalyzer.java
Modified:
james/mailbox/trunk/jcr/src/main/java/org/apache/james/mailbox/jcr/JCRMailboxSessionMapperFactory.java
james/mailbox/trunk/jpa/src/main/java/org/apache/james/mailbox/jpa/JPAMailboxSessionMapperFactory.java
james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirMailboxSessionMapperFactory.java
james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/model/MaildirMessage.java
james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/MailboxSessionMapperFactory.java
james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/AbstractMessageMapper.java
james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/MessageMapper.java
james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LuceneMessageSearchIndex.java
james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/streaming/InputStreamContent.java
james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/LuceneMessageSearchIndexTest.java
Modified: james/mailbox/trunk/jcr/src/main/java/org/apache/james/mailbox/jcr/JCRMailboxSessionMapperFactory.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/jcr/src/main/java/org/apache/james/mailbox/jcr/JCRMailboxSessionMapperFactory.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/jcr/src/main/java/org/apache/james/mailbox/jcr/JCRMailboxSessionMapperFactory.java (original)
+++ james/mailbox/trunk/jcr/src/main/java/org/apache/james/mailbox/jcr/JCRMailboxSessionMapperFactory.java Wed Jun 1 16:49:05 2011
@@ -29,8 +29,6 @@ import org.apache.james.mailbox.store.ma
import org.apache.james.mailbox.store.mail.MessageMapper;
import org.apache.james.mailbox.store.search.MessageSearchIndex;
import org.apache.james.mailbox.store.user.SubscriptionMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* JCR implementation of a {@link MailboxSessionMapperFactory}
@@ -43,15 +41,14 @@ public class JCRMailboxSessionMapperFact
private final static int DEFAULT_SCALING = 2;
private final int scaling;
private int messageScaling;
- private final MessageSearchIndex<String> index;
public JCRMailboxSessionMapperFactory(final MailboxSessionJCRRepository repository) {
this(repository, null, DEFAULT_SCALING, JCRMessageMapper.MESSAGE_SCALE_DAY);
}
public JCRMailboxSessionMapperFactory(final MailboxSessionJCRRepository repository, final MessageSearchIndex<String> index, final int scaling, final int messageScaling) {
+ super(index);
this.repository = repository;
- this.index = index;
this.scaling = scaling;
this.messageScaling = messageScaling;
}
@@ -64,7 +61,7 @@ public class JCRMailboxSessionMapperFact
@Override
public MessageMapper<String> createMessageMapper(MailboxSession session) throws MailboxException {
- JCRMessageMapper messageMapper = new JCRMessageMapper(repository, session, index, messageScaling);
+ JCRMessageMapper messageMapper = new JCRMessageMapper(repository, session, getSearchIndex(), messageScaling);
return messageMapper;
}
Modified: james/mailbox/trunk/jpa/src/main/java/org/apache/james/mailbox/jpa/JPAMailboxSessionMapperFactory.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/jpa/src/main/java/org/apache/james/mailbox/jpa/JPAMailboxSessionMapperFactory.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/jpa/src/main/java/org/apache/james/mailbox/jpa/JPAMailboxSessionMapperFactory.java (original)
+++ james/mailbox/trunk/jpa/src/main/java/org/apache/james/mailbox/jpa/JPAMailboxSessionMapperFactory.java Wed Jun 1 16:49:05 2011
@@ -38,11 +38,10 @@ import org.apache.james.mailbox.store.us
public class JPAMailboxSessionMapperFactory extends MailboxSessionMapperFactory<Long> {
private final EntityManagerFactory entityManagerFactory;
- private MessageSearchIndex<Long> index;
public JPAMailboxSessionMapperFactory(MessageSearchIndex<Long> index, EntityManagerFactory entityManagerFactory) {
+ super(index);
this.entityManagerFactory = entityManagerFactory;
- this.index = index;
createEntityManager().close();
}
@@ -57,7 +56,7 @@ public class JPAMailboxSessionMapperFact
@Override
public MessageMapper<Long> createMessageMapper(MailboxSession session) {
- return new JPAMessageMapper(session, index, entityManagerFactory);
+ return new JPAMessageMapper(session, getSearchIndex(), entityManagerFactory);
}
@Override
Modified: james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirMailboxSessionMapperFactory.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirMailboxSessionMapperFactory.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirMailboxSessionMapperFactory.java (original)
+++ james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirMailboxSessionMapperFactory.java Wed Jun 1 16:49:05 2011
@@ -34,7 +34,6 @@ public class MaildirMailboxSessionMapper
MailboxSessionMapperFactory<Integer> {
private final MaildirStore store;
- private final MessageSearchIndex<Integer> index;
public MaildirMailboxSessionMapperFactory(MaildirStore store) {
this(store, null);
@@ -42,8 +41,8 @@ public class MaildirMailboxSessionMapper
public MaildirMailboxSessionMapperFactory(MaildirStore store, MessageSearchIndex<Integer> index) {
+ super(index);
this.store = store;
- this.index = index;
}
@@ -56,7 +55,7 @@ public class MaildirMailboxSessionMapper
@Override
protected MessageMapper<Integer> createMessageMapper(MailboxSession session)
throws MailboxException {
- return new MaildirMessageMapper(session, index, store);
+ return new MaildirMessageMapper(session, getSearchIndex(), store);
}
@Override
Modified: james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/model/MaildirMessage.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/model/MaildirMessage.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/model/MaildirMessage.java (original)
+++ james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/model/MaildirMessage.java Wed Jun 1 16:49:05 2011
@@ -28,7 +28,6 @@ import java.util.List;
import javax.mail.Flags;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.NotImplementedException;
import org.apache.james.mailbox.MailboxException;
import org.apache.james.mailbox.store.mail.model.Header;
import org.apache.james.mailbox.store.mail.model.Mailbox;
@@ -203,15 +202,6 @@ public class MaildirMessage extends Abst
this.internalDate = date;
}
- /*
- * (non-Javadoc)
- * @see org.apache.james.mailbox.store.mail.model.MailboxMembership#getMailboxId()
- */
- public Integer getMailboxId() {
- throw new NotImplementedException();
- }
-
-
@Override
public void setUid(long uid) {
modified = true;
Modified: james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/MailboxSessionMapperFactory.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/MailboxSessionMapperFactory.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/MailboxSessionMapperFactory.java (original)
+++ james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/MailboxSessionMapperFactory.java Wed Jun 1 16:49:05 2011
@@ -26,6 +26,8 @@ import org.apache.james.mailbox.store.ma
import org.apache.james.mailbox.store.mail.MailboxMapperFactory;
import org.apache.james.mailbox.store.mail.MessageMapper;
import org.apache.james.mailbox.store.mail.MessageMapperFactory;
+import org.apache.james.mailbox.store.search.LazyMessageSearchIndex;
+import org.apache.james.mailbox.store.search.MessageSearchIndex;
import org.apache.james.mailbox.store.transaction.Mapper;
import org.apache.james.mailbox.store.user.SubscriptionMapper;
import org.apache.james.mailbox.store.user.SubscriptionMapperFactory;
@@ -39,8 +41,24 @@ public abstract class MailboxSessionMapp
protected final static String MESSAGEMAPPER ="MESSAGEMAPPER";
protected final static String MAILBOXMAPPER ="MAILBOXMAPPER";
protected final static String SUBSCRIPTIONMAPPER ="SUBSCRIPTIONMAPPER";
+ private MessageSearchIndex<Id> index;
+ public MailboxSessionMapperFactory(MessageSearchIndex<Id> index) {
+ this.index = index;
+ if (index != null && index instanceof LazyMessageSearchIndex<?>) {
+ ((LazyMessageSearchIndex<Id>) index).setMessageMapperFactory(this);
+ }
+ }
+ public MailboxSessionMapperFactory() {
+ this(null);
+ }
+
+ protected MessageSearchIndex<Id> getSearchIndex() {
+ return index;
+ }
+
+
/*
* (non-Javadoc)
* @see org.apache.james.mailbox.store.MessageMapperFactory#getMessageMapper(org.apache.james.mailbox.MailboxSession)
Modified: james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/AbstractMessageMapper.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/AbstractMessageMapper.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/AbstractMessageMapper.java (original)
+++ james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/AbstractMessageMapper.java Wed Jun 1 16:49:05 2011
@@ -252,12 +252,21 @@ public abstract class AbstractMessageMap
* (non-Javadoc)
* @see org.apache.james.mailbox.store.mail.MessageMapper#add(org.apache.james.mailbox.store.mail.model.Mailbox, org.apache.james.mailbox.store.mail.model.Message)
*/
- public MessageMetaData add(Mailbox<Id> mailbox, Message<Id> message) throws MailboxException {
+ public MessageMetaData add(final Mailbox<Id> mailbox, Message<Id> message) throws MailboxException {
message.setUid(nextUid(mailbox));
message.setModSeq(nextModSeq(mailbox));
MessageMetaData data = save(mailbox, message);
if (index != null) {
- index.add(mailboxSession, mailbox, message);
+ findInMailbox(mailbox, MessageRange.one(data.getUid()), new MailboxMembershipCallback<Id>() {
+
+ @Override
+ public void onMailboxMembers(List<Message<Id>> list) throws MailboxException {
+ if (!list.isEmpty()) {
+ index.add(mailboxSession, mailbox, list.get(0));
+ }
+ }
+ });
+
}
return data;
Modified: james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/MessageMapper.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/MessageMapper.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/MessageMapper.java (original)
+++ james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/MessageMapper.java Wed Jun 1 16:49:05 2011
@@ -25,7 +25,6 @@ import java.util.Map;
import javax.mail.Flags;
import org.apache.james.mailbox.MailboxException;
-import org.apache.james.mailbox.MailboxSession;
import org.apache.james.mailbox.MessageMetaData;
import org.apache.james.mailbox.MessageRange;
import org.apache.james.mailbox.SearchQuery;
Added: james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/LazyMessageSearchIndex.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/LazyMessageSearchIndex.java?rev=1130220&view=auto
==============================================================================
--- james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/LazyMessageSearchIndex.java (added)
+++ james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/LazyMessageSearchIndex.java Wed Jun 1 16:49:05 2011
@@ -0,0 +1,126 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+package org.apache.james.mailbox.store.search;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.mail.Flags;
+
+import org.apache.james.mailbox.MailboxException;
+import org.apache.james.mailbox.MailboxSession;
+import org.apache.james.mailbox.MessageRange;
+import org.apache.james.mailbox.SearchQuery;
+import org.apache.james.mailbox.store.mail.MessageMapperFactory;
+import org.apache.james.mailbox.store.mail.model.Mailbox;
+import org.apache.james.mailbox.store.mail.model.Message;
+import org.apache.james.mailbox.store.transaction.Mapper.MailboxMembershipCallback;
+
+/**
+ * {@link MessageSearchIndex} implementation which wraps another {@link MessageSearchIndex} and will forward all calls to it.
+ *
+ * The only special thing about this is that it will index all the mails in the mailbox on the first call of {@link #search(MailboxSession, Mailbox, SearchQuery)}
+ *
+ * This class is mostly useful for in-memory indexes or for indexed that should be recreated on every server restart.
+ *
+ *
+ * @param <Id>
+ */
+public class LazyMessageSearchIndex<Id> implements MessageSearchIndex<Id> {
+
+ private MessageSearchIndex<Id> index;
+ private MessageMapperFactory<Id> factory;
+ private final ConcurrentHashMap<Id, Boolean> indexed = new ConcurrentHashMap<Id, Boolean>();
+
+
+ public LazyMessageSearchIndex(MessageSearchIndex<Id> index) {
+ this.index = index;
+ }
+
+
+ /**
+ * Set the {@link MessageMapperFactory} which will be used to access the {@link Message}'s of the {@link Mailbox} on first {@link #search(MailboxSession, Mailbox, SearchQuery)}
+ *
+ * This method must be called before the class can be used
+ *
+ * @param factory
+ */
+ public void setMessageMapperFactory(MessageMapperFactory<Id> factory) {
+ this.factory = factory;
+ }
+
+
+ @Override
+ public void add(MailboxSession session, Mailbox<Id> mailbox, Message<Id> message) throws MailboxException {
+ index.add(session, mailbox, message);
+ }
+
+ @Override
+ public void delete(MailboxSession session, Mailbox<Id> mailbox, MessageRange range) throws MailboxException {
+ index.delete(session, mailbox, range);
+ }
+
+ /**
+ * Lazy index the mailbox on first search request if it was not indexed before. After indexing is done it delegate the search request to the wrapped
+ * {@link MessageSearchIndex}. Be aware that concurrent search requests are blocked on the same "not-yet-indexed" mailbox till it the index process was
+ * complete
+ *
+ */
+ @Override
+ public Iterator<Long> search(final MailboxSession session, final Mailbox<Id> mailbox, SearchQuery searchQuery) throws MailboxException {
+ Id id = mailbox.getMailboxId();
+
+ Boolean done = indexed.get(id);
+ if (done == null) {
+ done = new Boolean(true);
+ Boolean oldDone = indexed.putIfAbsent(id, done);
+ if (oldDone != null) {
+ done = oldDone;
+ }
+ synchronized (done) {
+ factory.getMessageMapper(session).findInMailbox(mailbox, MessageRange.all(), new MailboxMembershipCallback<Id>() {
+
+ @Override
+ public void onMailboxMembers(List<Message<Id>> list) throws MailboxException {
+ for (int i = 0; i < list.size(); i++) {
+ final Message<Id> message = list.get(i);
+
+ try {
+ add(session, mailbox, message);
+ } catch (MailboxException e) {
+ session.getLog().debug("Unable to index message " + message.getUid() + " in mailbox " + mailbox.getName(), e);
+ }
+
+ }
+ }
+ });
+ }
+ }
+
+ return index.search(session, mailbox, searchQuery);
+ }
+
+
+ @Override
+ public void update(MailboxSession session, Mailbox<Id> mailbox, MessageRange range, Flags flags) throws MailboxException {
+ index.update(session, mailbox, range, flags);
+ }
+
+}
Added: james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LenientImapSearchAnalyzer.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LenientImapSearchAnalyzer.java?rev=1130220&view=auto
==============================================================================
--- james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LenientImapSearchAnalyzer.java (added)
+++ james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LenientImapSearchAnalyzer.java Wed Jun 1 16:49:05 2011
@@ -0,0 +1,55 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+package org.apache.james.mailbox.store.search.lucene;
+
+import java.io.Reader;
+
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.shingle.ShingleFilter;
+import org.apache.lucene.util.Version;
+
+/**
+ * This {@link Analyzer} is not 100% conform with RFC3501 but does
+ * most times exactly what the user would expect.
+ *
+ */
+public class LenientImapSearchAnalyzer extends Analyzer{
+
+ public final static int DEFAULT_MAX_TOKEN_LENGTH = 4;
+
+ private final int maxTokenLength;
+
+
+ public LenientImapSearchAnalyzer(int maxTokenLength) {
+ this.maxTokenLength = maxTokenLength;
+ }
+
+ public LenientImapSearchAnalyzer() {
+ this(DEFAULT_MAX_TOKEN_LENGTH);
+ }
+
+ @Override
+ public TokenStream tokenStream(String arg0, Reader reader) {
+ return new ShingleFilter(new LowerCaseFilter(Version.LUCENE_31, new WhitespaceTokenizer(Version.LUCENE_31, reader)), 2, maxTokenLength);
+ }
+}
Modified: james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LuceneMessageSearchIndex.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LuceneMessageSearchIndex.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LuceneMessageSearchIndex.java (original)
+++ james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LuceneMessageSearchIndex.java Wed Jun 1 16:49:05 2011
@@ -18,9 +18,11 @@
****************************************************************/
package org.apache.james.mailbox.store.search.lucene;
-import java.io.ByteArrayOutputStream;
+import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
@@ -36,6 +38,7 @@ import org.apache.james.mailbox.MailboxE
import org.apache.james.mailbox.MailboxSession;
import org.apache.james.mailbox.MessageRange;
import org.apache.james.mailbox.SearchQuery;
+import org.apache.james.mailbox.UnsupportedSearchException;
import org.apache.james.mailbox.SearchQuery.AllCriterion;
import org.apache.james.mailbox.SearchQuery.ContainsOperator;
import org.apache.james.mailbox.SearchQuery.Criterion;
@@ -47,7 +50,6 @@ import org.apache.james.mailbox.SearchQu
import org.apache.james.mailbox.SearchQuery.NumericOperator;
import org.apache.james.mailbox.SearchQuery.NumericRange;
import org.apache.james.mailbox.SearchQuery.UidCriterion;
-import org.apache.james.mailbox.UnsupportedSearchException;
import org.apache.james.mailbox.store.mail.model.Mailbox;
import org.apache.james.mailbox.store.mail.model.Message;
import org.apache.james.mailbox.store.search.MessageSearchIndex;
@@ -60,14 +62,14 @@ import org.apache.james.mime4j.field.add
import org.apache.james.mime4j.field.address.MailboxList;
import org.apache.james.mime4j.message.Header;
import org.apache.james.mime4j.message.SimpleContentHandler;
+import org.apache.james.mime4j.parser.MimeEntityConfig;
import org.apache.james.mime4j.parser.MimeStreamParser;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericField;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.NumericField;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
@@ -84,12 +86,14 @@ import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.Version;
/**
- * Lucene based {@link MessageSearchIndex} which offers message searching.
+ * Lucene based {@link MessageSearchIndex} which offers message searching via a Lucene index
+ *
*
* @param <Id>
@@ -128,6 +132,9 @@ public class LuceneMessageSearchIndex<Id
public final static String BODY_FIELD = "body";
+ /**
+ * Prefix which will be used for each message header to store it also in a seperate {@link Field}
+ */
public final static String PREFIX_HEADER_FIELD ="header_";
/**
@@ -135,22 +142,68 @@ public class LuceneMessageSearchIndex<Id
*/
public final static String HEADERS_FIELD ="headers";
+ /**
+ * {@link Field} which will contain the mod-sequence of the message
+ */
public final static String MODSEQ_FIELD = "modSeq";
-
+ /**
+ * {@link Field} which will contain the TO-Address of the message
+ */
public final static String TO_FIELD ="to";
+
+
+ /**
+ * {@link Field} which will contain the CC-Address of the message
+ */
public final static String CC_FIELD ="cc";
- public final static String BCC_FIELD ="bcc";
- public final static String FROM_FIELD ="from";
+ /**
+ * {@link Field} which will contain the BCC-Address of the message
+ */
+ public final static String BCC_FIELD ="bcc";
+
+ /**
+ * {@link Field} which will contain the FROM-Address of the message
+ */
+ public final static String FROM_FIELD ="from";
+
+ /**
+ * {@link Field} which contain the internalDate of the message with YEAR-Resolution
+ */
public final static String INTERNAL_DATE_FIELD_YEAR_RESOLUTION ="internaldateYearResolution";
-
+
+
+ /**
+ * {@link Field} which contain the internalDate of the message with MONTH-Resolution
+ */
public final static String INTERNAL_DATE_FIELD_MONTH_RESOLUTION ="internaldateMonthResolution";
+
+ /**
+ * {@link Field} which contain the internalDate of the message with DAY-Resolution
+ */
public final static String INTERNAL_DATE_FIELD_DAY_RESOLUTION ="internaldateDayResolution";
+
+ /**
+ * {@link Field} which contain the internalDate of the message with HOUR-Resolution
+ */
public final static String INTERNAL_DATE_FIELD_HOUR_RESOLUTION ="internaldateHourResolution";
+
+ /**
+ * {@link Field} which contain the internalDate of the message with MINUTE-Resolution
+ */
public final static String INTERNAL_DATE_FIELD_MINUTE_RESOLUTION ="internaldateMinuteResolution";
+
+ /**
+ * {@link Field} which contain the internalDate of the message with SECOND-Resolution
+ */
public final static String INTERNAL_DATE_FIELD_SECOND_RESOLUTION ="internaldateSecondResolution";
+
+
+ /**
+ * {@link Field} which contain the internalDate of the message with MILLISECOND-Resolution
+ */
public final static String INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION ="internaldateMillisecondResolution";
/**
@@ -159,17 +212,26 @@ public class LuceneMessageSearchIndex<Id
public final static String MAILBOX_ID_FIELD ="mailboxid";
+
private final static String MEDIA_TYPE_TEXT = "text";
private final static String MEDIA_TYPE_MESSAGE = "message";
-
+ private final static String DEFAULT_ENCODING = "US-ASCII";
+
private final IndexWriter writer;
private int maxQueryResults = DEFAULT_MAX_QUERY_RESULTS;
+
+ private boolean suffixMatch = false;
private final static Sort UID_SORT = new Sort(new SortField(UID_FIELD, SortField.LONG));
- public LuceneMessageSearchIndex(Directory directory, boolean breakIMAPRFC) throws CorruptIndexException, LockObtainFailedException, IOException {
- this(new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_31, createAnalyzer(breakIMAPRFC))));
+ public LuceneMessageSearchIndex(Directory directory) throws CorruptIndexException, LockObtainFailedException, IOException {
+ this(directory, true);
+ }
+
+
+ public LuceneMessageSearchIndex(Directory directory, boolean lenient) throws CorruptIndexException, LockObtainFailedException, IOException {
+ this(new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_31, createAnalyzer(lenient))));
}
@@ -178,7 +240,7 @@ public class LuceneMessageSearchIndex<Id
}
/**
- * Set the max count of results which will get returned from a query
+ * Set the max count of results which will get returned from a query. The default is {@link #DEFAULT_MAX_QUERY_RESULTS}
*
* @param maxQueryResults
*/
@@ -188,19 +250,33 @@ public class LuceneMessageSearchIndex<Id
/**
* Create a {@link Analyzer} which is used to index the {@link MailboxMembership}'s
*
- * @param breakIMAPRFC
+ * @param lenient
*
* @return analyzer
*/
- private static Analyzer createAnalyzer(boolean breakIMAPRFC) {
- if (breakIMAPRFC) {
- return new SimpleAnalyzer(Version.LUCENE_31);
+ private static Analyzer createAnalyzer(boolean lenient) {
+ if (lenient) {
+ return new LenientImapSearchAnalyzer();
} else {
- return new ImapSearchAnalyzer();
+ return new StrictImapSearchAnalyzer();
}
-
}
+ /**
+ * If set to true this implementation will use {@link WildcardQuery} to match suffix and prefix. This is what RFC3501 expects but is often not what the user does.
+ * It also slow things a lot if you have complex queries which use many "TEXT" arguments. If you want the implementation to behave strict like RFC3501 says, you should
+ * set this to true.
+ *
+ * The default is false for performance reasons
+ *
+ *
+ * @param suffixMatch
+ */
+ public void setEnableSuffixMatch(boolean suffixMatch) {
+ this.suffixMatch = suffixMatch;
+ }
+
+
/*
* (non-Javadoc)
@@ -237,6 +313,7 @@ public class LuceneMessageSearchIndex<Id
return uids.iterator();
}
+
/**
* Create a new {@link Document} for the given {@link MailboxMembership}. This Document does not contain any flags data. The {@link Flags} are stored in a seperate Document.
*
@@ -253,8 +330,6 @@ public class LuceneMessageSearchIndex<Id
// create an unqiue key for the document which can be used later on updates to find the document
doc.add(new Field(ID_FIELD, membership.getMailboxId().toString().toLowerCase(Locale.US) +"-" + Long.toString(membership.getUid()), Store.YES, Index.NOT_ANALYZED));
-
-
doc.add(new NumericField(INTERNAL_DATE_FIELD_YEAR_RESOLUTION,Store.NO, true).setLongValue(DateUtils.truncate(membership.getInternalDate(),Calendar.YEAR).getTime()));
doc.add(new NumericField(INTERNAL_DATE_FIELD_MONTH_RESOLUTION,Store.NO, true).setLongValue(DateUtils.truncate(membership.getInternalDate(),Calendar.MONTH).getTime()));
@@ -265,14 +340,11 @@ public class LuceneMessageSearchIndex<Id
doc.add(new NumericField(INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION,Store.NO, true).setLongValue(DateUtils.truncate(membership.getInternalDate(),Calendar.MILLISECOND).getTime()));
doc.add(new NumericField(SIZE_FIELD,Store.NO, true).setLongValue(membership.getFullContentOctets()));
-
+
// content handler which will index the headers and the body of the message
SimpleContentHandler handler = new SimpleContentHandler() {
-
- /**
- * Add the headers to the Document
- */
+
public void headers(Header header) {
Iterator<org.apache.james.mime4j.parser.Field> fields = header.iterator();
@@ -321,32 +393,42 @@ public class LuceneMessageSearchIndex<Id
}
}
-
-
-
- /**
- * Add the body parts to the Document
+ /*
+ * (non-Javadoc)
+ * @see org.apache.james.mime4j.message.SimpleContentHandler#bodyDecoded(org.apache.james.mime4j.descriptor.BodyDescriptor, java.io.InputStream)
*/
public void bodyDecoded(BodyDescriptor desc, InputStream in) throws IOException {
String mediaType = desc.getMediaType();
- String charset = desc.getCharset();
if (MEDIA_TYPE_TEXT.equalsIgnoreCase(mediaType) || MEDIA_TYPE_MESSAGE.equalsIgnoreCase(mediaType)) {
- // TODO: maybe we want to limit the length here ?
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- int b = -1;
- while ((b = in.read()) != -1) {
- out.write(b);
+ String cset = desc.getCharset();
+ if (cset == null) {
+ cset = DEFAULT_ENCODING;
+ }
+ Charset charset;
+ try {
+ charset = Charset.forName(cset);
+ } catch (Exception e) {
+ // Invalid charset found so fallback toe the DEFAULT_ENCODING
+ charset = Charset.forName(DEFAULT_ENCODING);
+ }
+
+ // Read the content one line after the other and add it to the document
+ BufferedReader bodyReader = new BufferedReader(new InputStreamReader(in, charset));
+ String line = null;
+ while((line = bodyReader.readLine()) != null) {
+ doc.add(new Field(BODY_FIELD, line.toLowerCase(Locale.US),Store.NO, Index.ANALYZED));
}
- out.flush();
- doc.add(new Field(BODY_FIELD, out.toString(charset).toLowerCase(Locale.US),Store.NO, Index.ANALYZED));
- out.close();
}
}
};
-
- MimeStreamParser parser = new MimeStreamParser();
+ MimeEntityConfig config = new MimeEntityConfig();
+ config.setMaxLineLen(-1);
+ config.setStrictParsing(false);
+ config.setMaxContentLen(-1);
+ MimeStreamParser parser = new MimeStreamParser(config);
parser.setContentHandler(handler);
+
try {
// parse the message to index headers and body
parser.parse(membership.getFullContent());
@@ -358,7 +440,7 @@ public class LuceneMessageSearchIndex<Id
// anyway let us just skip the body and headers in the index
throw new MailboxException("Unable to index content of message", e);
}
-
+
return doc;
}
@@ -457,6 +539,20 @@ public class LuceneMessageSearchIndex<Id
}
/**
+ * This method will return the right {@link Query} depending if {@link #suffixMatch} is enabled
+ *
+ * @param fieldName
+ * @param value
+ * @return query
+ */
+ private Query createTermQuery(String fieldName, String value) {
+ if (suffixMatch) {
+ return new WildcardQuery(new Term(fieldName, "*" + value + "*"));
+ } else {
+ return new PrefixQuery(new Term(fieldName, value));
+ }
+ }
+ /**
* Return a {@link Query} which is build based on the given {@link SearchQuery.HeaderCriterion}
*
* @param crit
@@ -468,11 +564,11 @@ public class LuceneMessageSearchIndex<Id
String fieldName = PREFIX_HEADER_FIELD + crit.getHeaderName().toLowerCase(Locale.US);
if (op instanceof SearchQuery.ContainsOperator) {
ContainsOperator cop = (ContainsOperator) op;
- return new PrefixQuery(new Term(fieldName, cop.getValue().toLowerCase(Locale.US)));
+ return createTermQuery(fieldName, cop.getValue().toLowerCase(Locale.US));
} else if (op instanceof SearchQuery.ExistsOperator){
return new PrefixQuery(new Term(fieldName, ""));
} else if (op instanceof SearchQuery.AddressOperator) {
- return new PrefixQuery(new Term(fieldName.toLowerCase(), ((SearchQuery.AddressOperator) op).getAddress().toLowerCase(Locale.US)));
+ return createTermQuery(fieldName.toLowerCase(), ((SearchQuery.AddressOperator) op).getAddress().toLowerCase(Locale.US));
} else {
// Operator not supported
throw new UnsupportedSearchException();
@@ -611,13 +707,14 @@ public class LuceneMessageSearchIndex<Id
* @throws UnsupportedSearchException
*/
private Query createTextQuery(SearchQuery.TextCriterion crit) throws UnsupportedSearchException {
+ String value = crit.getOperator().getValue().toLowerCase(Locale.US);
switch(crit.getType()) {
case BODY:
- return new PrefixQuery(new Term(BODY_FIELD, crit.getOperator().getValue().toLowerCase(Locale.US)));
+ return createTermQuery(BODY_FIELD, value);
case FULL:
BooleanQuery query = new BooleanQuery();
- query.add(new PrefixQuery(new Term(BODY_FIELD, crit.getOperator().getValue().toLowerCase(Locale.US))), BooleanClause.Occur.SHOULD);
- query.add(new PrefixQuery(new Term(HEADERS_FIELD, crit.getOperator().getValue().toLowerCase(Locale.US))), BooleanClause.Occur.SHOULD);
+ query.add(createTermQuery(BODY_FIELD, value), BooleanClause.Occur.SHOULD);
+ query.add(createTermQuery(HEADERS_FIELD,value), BooleanClause.Occur.SHOULD);
return query;
default:
throw new UnsupportedSearchException();
Copied: james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/StrictImapSearchAnalyzer.java (from r1130081, james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/ImapSearchAnalyzer.java)
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/StrictImapSearchAnalyzer.java?p2=james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/StrictImapSearchAnalyzer.java&p1=james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/ImapSearchAnalyzer.java&r1=1130081&r2=1130220&rev=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/ImapSearchAnalyzer.java (original)
+++ james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/StrictImapSearchAnalyzer.java Wed Jun 1 16:49:05 2011
@@ -38,15 +38,15 @@ import org.apache.lucene.util.Version;
* case-insensitive.
*
*/
-public final class ImapSearchAnalyzer extends Analyzer {
+public final class StrictImapSearchAnalyzer extends Analyzer {
private final int minTokenLength;
private final int maxTokenLength;
- public ImapSearchAnalyzer() {
+ public StrictImapSearchAnalyzer() {
this(3, 40);
}
- public ImapSearchAnalyzer(int minTokenLength, int maxTokenLength) {
+ public StrictImapSearchAnalyzer(int minTokenLength, int maxTokenLength) {
this.minTokenLength = minTokenLength;
this.maxTokenLength = maxTokenLength;
}
Modified: james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/streaming/InputStreamContent.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/streaming/InputStreamContent.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/streaming/InputStreamContent.java (original)
+++ james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/streaming/InputStreamContent.java Wed Jun 1 16:49:05 2011
@@ -34,8 +34,7 @@ import org.apache.james.mailbox.store.ma
*
*/
public final class InputStreamContent implements org.apache.james.mailbox.InputStreamContent{
- @SuppressWarnings("unchecked")
- private Message m;
+ private Message<?> m;
private Type type;
public static enum Type {
@@ -43,8 +42,7 @@ public final class InputStreamContent im
Body
}
- @SuppressWarnings("unchecked")
- public InputStreamContent(Message m, Type type) throws IOException{
+ public InputStreamContent(Message<?> m, Type type) throws IOException{
this.m = m;
this.type = type;
}
Modified: james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/LuceneMessageSearchIndexTest.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/LuceneMessageSearchIndexTest.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/LuceneMessageSearchIndexTest.java (original)
+++ james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/LuceneMessageSearchIndexTest.java Wed Jun 1 16:49:05 2011
@@ -40,7 +40,6 @@ import org.apache.james.mailbox.store.Si
import org.apache.james.mailbox.store.SimpleMailboxMembership;
import org.apache.james.mailbox.store.mail.model.Mailbox;
import org.apache.james.mailbox.store.mail.model.Message;
-import org.apache.james.mailbox.store.search.MessageSearchIndex;
import org.apache.james.mailbox.store.search.lucene.LuceneMessageSearchIndex;
import org.apache.lucene.store.RAMDirectory;
import org.junit.Before;
@@ -48,7 +47,7 @@ import org.junit.Test;
public class LuceneMessageSearchIndexTest {
- private MessageSearchIndex<Long> index;
+ private LuceneMessageSearchIndex<Long> index;
private SimpleMailbox mailbox = new SimpleMailbox(0);
private SimpleMailbox mailbox2 = new SimpleMailbox(1);
@@ -69,9 +68,14 @@ public class LuceneMessageSearchIndexTes
Message<Long> row;
+ protected boolean useLenient() {
+ return true;
+ }
+
@Before
public void setUp() throws Exception {
- index = new LuceneMessageSearchIndex<Long>(new RAMDirectory(), false);
+ index = new LuceneMessageSearchIndex<Long>(new RAMDirectory(), useLenient());
+ index.setEnableSuffixMatch(true);
List<org.apache.james.mailbox.store.SimpleHeader> headersSubject = new ArrayList<org.apache.james.mailbox.store.SimpleHeader>();
headersSubject.add(new SimpleHeader("Subject", 1, "test"));
Added: james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/StrictLuceneMessageSearchIndexText.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/StrictLuceneMessageSearchIndexText.java?rev=1130220&view=auto
==============================================================================
--- james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/StrictLuceneMessageSearchIndexText.java (added)
+++ james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/StrictLuceneMessageSearchIndexText.java Wed Jun 1 16:49:05 2011
@@ -0,0 +1,28 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+package org.apache.james.mailbox.store.lucene;
+
+public class StrictLuceneMessageSearchIndexText extends LuceneMessageSearchIndexTest{
+
+ @Override
+ protected boolean useLenient() {
+ return false;
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org