You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@james.apache.org by rc...@apache.org on 2024/11/06 03:43:59 UTC

(james-project) branch master updated (afb9246d74 -> 164404f9b8)

This is an automated email from the ASF dual-hosted git repository.

rcordier pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git


    from afb9246d74 JAMES-4084 Implement webadmin to list inactive users (#2474)
     new b4e7f4cda4 JAMES-4082 Move MimePartParser and related classes from the `opensearch` module to the `mailbox-store` module
     new a4191571ef JAMES-4082 Introduce LuceneIndexableDocument
     new 956d5cc8b1 JAMES-4082 Adapt Lucene search - support full capability
     new fcc410b70e JAMES-4082 Lucene search - remove unused code & clean
     new cf4a4cc421 JAMES-4082 Update bean for LuceneMessageSearchIndex (Spring/JPA app)
     new f4912da7b3 JAMES-2182 Lucene search - remove SearchCapabilities.PartialEmailMatch (performance reason)
     new 164404f9b8 JAMES-2182 Lucene search highlight - moving test case `shouldHighlightAttachmentTextContentWhenTextBodyDoesNotMatch` to contract class

The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../searchhighligt/SearchHighLighterContract.java  |  39 ++
 mailbox/lucene/pom.xml                             |  11 +
 .../lucene/search/DocumentFieldConstants.java      | 253 +++++++++
 .../lucene/search/LuceneIndexableDocument.java     | 289 ++++++++++
 .../lucene/search/LuceneMessageSearchIndex.java    | 589 +++------------------
 .../lucene/search/LuceneSearchHighlighter.java     |  24 +-
 .../META-INF/spring/mailbox-index-lucene.xml       |   2 +
 .../search/LuceneFSSearchHighLighterTest.java      |   3 +-
 .../search/LuceneMailboxMessageFlagSearchTest.java |   5 +-
 .../LuceneMailboxMessageSearchIndexTest.java       |   3 +-
 .../search/LuceneMemorySearchHighLighterTest.java  |   4 +-
 .../search/LuceneMessageSearchIndexTest.java       |  25 +-
 .../{SerializableMessage.java => EMailerDto.java}  |   8 +-
 .../mailbox/opensearch/json/EMailersDto.java}      |  24 +-
 .../{SerializableMessage.java => HeaderDto.java}   |  11 +-
 .../mailbox/opensearch/json/IndexableMessage.java  |  52 +-
 .../mailbox/opensearch/json/MimePartDto.java}      |  31 +-
 .../{SerializableMessage.java => SubjectsDto.java} |   6 +-
 .../opensearch/query/CriterionConverter.java       |   2 +-
 .../opensearch/json/IndexableMessageTest.java      |   2 +-
 .../search/OpenSearchSearchHighlighterTest.java    |  48 --
 .../META-INF/spring/mailbox-index-lucene-test.xml  |   2 +
 .../james/mailbox/store/search/mime}/EMailer.java  |   6 +-
 .../james/mailbox/store/search/mime}/EMailers.java |   4 +-
 .../store/search/mime}/HeaderCollection.java       |   5 +-
 .../james/mailbox/store/search/mime}/MimePart.java |  15 +-
 .../search/mime}/MimePartContainerBuilder.java     |   2 +-
 .../mailbox/store/search/mime}/MimePartParser.java |   2 +-
 .../search/mime}/RootMimePartContainerBuilder.java |   2 +-
 .../store/search/mime}/SerializableMessage.java    |   2 +-
 .../james/mailbox/store/search/mime}/Subjects.java |   4 +-
 .../mailbox/store/search/mime}/EMailerTest.java    |   2 +-
 .../mailbox/store/search/mime}/EMailersTest.java   |   2 +-
 .../mailbox/store/search/mime}/FieldImpl.java      |   2 +-
 .../store/search/mime}/HeaderCollectionTest.java   |   2 +-
 .../mailbox/store/search/mime}/MimePartTest.java   |   2 +-
 .../mailbox/store/search/mime}/SubjectsTest.java   |   2 +-
 .../lucenesearch/host/LuceneSearchHostSystem.java  |   3 +-
 .../java/org/apache/james/JPAJamesServerMain.java  |   8 +-
 39 files changed, 825 insertions(+), 673 deletions(-)
 create mode 100644 mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/DocumentFieldConstants.java
 create mode 100644 mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneIndexableDocument.java
 copy mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/{SerializableMessage.java => EMailerDto.java} (77%)
 copy mailbox/opensearch/src/{test/java/org/apache/james/mailbox/opensearch/json/EMailerTest.java => main/java/org/apache/james/mailbox/opensearch/json/EMailersDto.java} (71%)
 copy mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/{SerializableMessage.java => HeaderDto.java} (73%)
 copy mailbox/opensearch/src/{test/java/org/apache/james/mailbox/opensearch/json/EMailerTest.java => main/java/org/apache/james/mailbox/opensearch/json/MimePartDto.java} (53%)
 copy mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/{SerializableMessage.java => SubjectsDto.java} (90%)
 rename mailbox/{opensearch/src/main/java/org/apache/james/mailbox/opensearch/json => store/src/main/java/org/apache/james/mailbox/store/search/mime}/EMailer.java (91%)
 rename mailbox/{opensearch/src/main/java/org/apache/james/mailbox/opensearch/json => store/src/main/java/org/apache/james/mailbox/store/search/mime}/EMailers.java (94%)
 rename mailbox/{opensearch/src/main/java/org/apache/james/mailbox/opensearch/json => store/src/main/java/org/apache/james/mailbox/store/search/mime}/HeaderCollection.java (97%)
 rename mailbox/{opensearch/src/main/java/org/apache/james/mailbox/opensearch/json => store/src/main/java/org/apache/james/mailbox/store/search/mime}/MimePart.java (95%)
 rename mailbox/{opensearch/src/main/java/org/apache/james/mailbox/opensearch/json => store/src/main/java/org/apache/james/mailbox/store/search/mime}/MimePartContainerBuilder.java (97%)
 rename mailbox/{opensearch/src/main/java/org/apache/james/mailbox/opensearch/json => store/src/main/java/org/apache/james/mailbox/store/search/mime}/MimePartParser.java (99%)
 rename mailbox/{opensearch/src/main/java/org/apache/james/mailbox/opensearch/json => store/src/main/java/org/apache/james/mailbox/store/search/mime}/RootMimePartContainerBuilder.java (98%)
 rename mailbox/{opensearch/src/main/java/org/apache/james/mailbox/opensearch/json => store/src/main/java/org/apache/james/mailbox/store/search/mime}/SerializableMessage.java (96%)
 rename mailbox/{opensearch/src/main/java/org/apache/james/mailbox/opensearch/json => store/src/main/java/org/apache/james/mailbox/store/search/mime}/Subjects.java (94%)
 rename mailbox/{opensearch/src/test/java/org/apache/james/mailbox/opensearch/json => store/src/test/java/org/apache/james/mailbox/store/search/mime}/EMailerTest.java (97%)
 rename mailbox/{opensearch/src/test/java/org/apache/james/mailbox/opensearch/json => store/src/test/java/org/apache/james/mailbox/store/search/mime}/EMailersTest.java (98%)
 rename mailbox/{opensearch/src/test/java/org/apache/james/mailbox/opensearch/json => store/src/test/java/org/apache/james/mailbox/store/search/mime}/FieldImpl.java (97%)
 rename mailbox/{opensearch/src/test/java/org/apache/james/mailbox/opensearch/json => store/src/test/java/org/apache/james/mailbox/store/search/mime}/HeaderCollectionTest.java (99%)
 rename mailbox/{opensearch/src/test/java/org/apache/james/mailbox/opensearch/json => store/src/test/java/org/apache/james/mailbox/store/search/mime}/MimePartTest.java (97%)
 rename mailbox/{opensearch/src/test/java/org/apache/james/mailbox/opensearch/json => store/src/test/java/org/apache/james/mailbox/store/search/mime}/SubjectsTest.java (97%)


---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org


(james-project) 05/07: JAMES-4082 Update bean for LuceneMessageSearchIndex (Spring/JPA app)

Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git

commit cf4a4cc42116aec94b807f59ffe73145549aede6
Author: TungTV <vt...@linagora.com>
AuthorDate: Mon Nov 4 08:29:24 2024 +0700

    JAMES-4082 Update bean for LuceneMessageSearchIndex (Spring/JPA app)
---
 .../src/main/resources/META-INF/spring/mailbox-index-lucene.xml   | 2 ++
 .../test/resources/META-INF/spring/mailbox-index-lucene-test.xml  | 2 ++
 .../src/main/java/org/apache/james/JPAJamesServerMain.java        | 8 +++++++-
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/mailbox/lucene/src/main/resources/META-INF/spring/mailbox-index-lucene.xml b/mailbox/lucene/src/main/resources/META-INF/spring/mailbox-index-lucene.xml
index 16d332ef32..27f7e5e767 100644
--- a/mailbox/lucene/src/main/resources/META-INF/spring/mailbox-index-lucene.xml
+++ b/mailbox/lucene/src/main/resources/META-INF/spring/mailbox-index-lucene.xml
@@ -38,8 +38,10 @@
         <constructor-arg index="2" type="org.apache.lucene.store.Directory" ref="fsDirectory"/>
         <constructor-arg index="3" ref="messageIdFactory"/>
         <constructor-arg index="4" ref="sessionProvider"/>
+        <constructor-arg index="5" ref="textExtractor"/>
         <property name="enableSuffixMatch" value="true"/>
     </bean>
+    <bean id="textExtractor" class="org.apache.james.mailbox.store.extractor.DefaultTextExtractor"/>
     <bean id="fsDirectory"  class="org.apache.lucene.store.FSDirectory" factory-method="open">
         <constructor-arg index="0" value="../var/store/lucene"/>
     </bean>
diff --git a/mailbox/spring/src/test/resources/META-INF/spring/mailbox-index-lucene-test.xml b/mailbox/spring/src/test/resources/META-INF/spring/mailbox-index-lucene-test.xml
index bf7036e3c9..b7a05fb6ed 100644
--- a/mailbox/spring/src/test/resources/META-INF/spring/mailbox-index-lucene-test.xml
+++ b/mailbox/spring/src/test/resources/META-INF/spring/mailbox-index-lucene-test.xml
@@ -33,11 +33,13 @@
         <constructor-arg index="2" type="org.apache.lucene.store.Directory" ref="fsDirectory"/>
         <constructor-arg index="3" ref="messageIdFactory"/>
         <constructor-arg index="4" ref="sessionProvider"/>
+        <constructor-arg index="5" ref="textExtractor"/>
         <property name="enableSuffixMatch" value="true"/>
     </bean>
     <bean id="fsDirectory"  class="org.apache.lucene.store.FSDirectory" factory-method="open">
         <constructor-arg index="0" value="../var/store/lucene"/>
     </bean>
+    <bean id="textExtractor" class="org.apache.james.mailbox.store.extractor.DefaultTextExtractor"/>
     <alias name="jpa-mailboxIdFactory" alias="mailboxIdFactory"/>
     <alias name="jpa-mailboxmanager" alias="mailboxmanager"/>
     <alias name="luceneIndex" alias="indexer"/>
diff --git a/server/apps/jpa-app/src/main/java/org/apache/james/JPAJamesServerMain.java b/server/apps/jpa-app/src/main/java/org/apache/james/JPAJamesServerMain.java
index aa0b291252..b41bd72e62 100644
--- a/server/apps/jpa-app/src/main/java/org/apache/james/JPAJamesServerMain.java
+++ b/server/apps/jpa-app/src/main/java/org/apache/james/JPAJamesServerMain.java
@@ -20,6 +20,8 @@
 package org.apache.james;
 
 import org.apache.james.data.UsersRepositoryModuleChooser;
+import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.store.extractor.JsoupTextExtractor;
 import org.apache.james.modules.MailboxModule;
 import org.apache.james.modules.MailetProcessingModule;
 import org.apache.james.modules.RunArgumentsModule;
@@ -82,7 +84,12 @@ public class JPAJamesServerMain implements JamesServerMain {
         new SMTPServerModule(),
         WEBADMIN);
 
+    private static final Module SEARCH_MODULE = Modules.combine(
+        new LuceneSearchMailboxModule(),
+        binder -> binder.bind(TextExtractor.class).toInstance(new JsoupTextExtractor()));
+
     private static final Module JPA_SERVER_MODULE = Modules.combine(
+        SEARCH_MODULE,
         new ActiveMQQueueModule(),
         new NaiveDelegationStoreModule(),
         new DefaultProcessorsConfigurationProviderModule(),
@@ -90,7 +97,6 @@ public class JPAJamesServerMain implements JamesServerMain {
         new JPAMailboxModule(),
         new MailboxModule(),
         new ReIndexingTaskSerializationModule(),
-        new LuceneSearchMailboxModule(),
         new NoJwtModule(),
         new RawPostDequeueDecoratorModule(),
         new SieveJPARepositoryModules(),


---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org


(james-project) 07/07: JAMES-2182 Lucene search highlight - moving test case `shouldHighlightAttachmentTextContentWhenTextBodyDoesNotMatch` to contract class

Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git

commit 164404f9b86a0a5f458f1041646ede9f687fbcfb
Author: TungTV <vt...@linagora.com>
AuthorDate: Mon Nov 4 15:04:38 2024 +0700

    JAMES-2182 Lucene search highlight - moving test case `shouldHighlightAttachmentTextContentWhenTextBodyDoesNotMatch` to contract class
---
 .../searchhighligt/SearchHighLighterContract.java  | 39 ++++++++++++++++
 .../search/LuceneMemorySearchHighLighterTest.java  | 54 ----------------------
 .../search/OpenSearchSearchHighlighterTest.java    | 48 -------------------
 3 files changed, 39 insertions(+), 102 deletions(-)

diff --git a/mailbox/api/src/test/java/org/apache/james/mailbox/searchhighligt/SearchHighLighterContract.java b/mailbox/api/src/test/java/org/apache/james/mailbox/searchhighligt/SearchHighLighterContract.java
index e273e95e9c..8083639e87 100644
--- a/mailbox/api/src/test/java/org/apache/james/mailbox/searchhighligt/SearchHighLighterContract.java
+++ b/mailbox/api/src/test/java/org/apache/james/mailbox/searchhighligt/SearchHighLighterContract.java
@@ -37,6 +37,7 @@ import org.apache.james.mailbox.model.MessageId;
 import org.apache.james.mailbox.model.MultimailboxesSearchQuery;
 import org.apache.james.mailbox.model.SearchQuery;
 import org.apache.james.mime4j.dom.Message;
+import org.apache.james.util.ClassLoaderUtils;
 import org.junit.jupiter.api.Test;
 
 import reactor.core.publisher.Flux;
@@ -465,4 +466,42 @@ public interface SearchHighLighterContract {
             .collectList()
             .block()).hasSize(0);
     }
+
+    @Test
+    default void shouldHighlightAttachmentTextContentWhenTextBodyDoesNotMatch() throws Exception {
+        MailboxSession session = session(USERNAME1);
+
+        ComposedMessageId m1 = appendMessage(MessageManager.AppendCommand.from(
+                Message.Builder.of()
+                    .setTo("to@james.local")
+                    .setSubject("Hallo, Thx Matthieu for your help")
+                    .setBody("append contentA to inbox", StandardCharsets.UTF_8)),
+            session).getId();
+
+        // m2 has an attachment with text content: "This is a beautiful banana"
+        ComposedMessageId m2 = appendMessage(
+            MessageManager.AppendCommand.builder()
+                .build(ClassLoaderUtils.getSystemResourceAsSharedStream("eml/emailWithTextAttachment.eml")),
+            session).getId();
+
+        verifyMessageWasIndexed(2);
+
+        String keywordSearch = "beautiful";
+        MultimailboxesSearchQuery multiMailboxSearch = MultimailboxesSearchQuery.from(SearchQuery.of(
+                new SearchQuery.ConjunctionCriterion(SearchQuery.Conjunction.OR,
+                    List.of(SearchQuery.bodyContains(keywordSearch),
+                        SearchQuery.attachmentContains(keywordSearch)))))
+            .inMailboxes(List.of(m1.getMailboxId(), m2.getMailboxId()))
+            .build();
+
+        List<SearchSnippet> searchSnippets = Flux.from(testee().highlightSearch(List.of(m1.getMessageId(), m2.getMessageId()), multiMailboxSearch, session))
+            .collectList()
+            .block();
+
+        assertThat(searchSnippets).hasSize(1);
+
+        assertThat(searchSnippets.getFirst().highlightedBody())
+            .isPresent()
+            .satisfies(highlightedBody -> assertThat(highlightedBody.get()).contains("This is a <mark>beautiful</mark> banana"));
+    }
 }
diff --git a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMemorySearchHighLighterTest.java b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMemorySearchHighLighterTest.java
index 23af88f0b8..1ac768d18a 100644
--- a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMemorySearchHighLighterTest.java
+++ b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMemorySearchHighLighterTest.java
@@ -19,45 +19,32 @@
 package org.apache.james.mailbox.lucene.search;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.jupiter.api.Assumptions.assumeTrue;
-
-import java.nio.charset.StandardCharsets;
-import java.util.List;
 
 import org.apache.james.core.Username;
-import org.apache.james.mailbox.MailboxManager;
 import org.apache.james.mailbox.MailboxSession;
 import org.apache.james.mailbox.MessageManager;
 import org.apache.james.mailbox.exception.MailboxException;
 import org.apache.james.mailbox.inmemory.InMemoryId;
 import org.apache.james.mailbox.inmemory.InMemoryMessageId;
 import org.apache.james.mailbox.inmemory.manager.InMemoryIntegrationResources;
-import org.apache.james.mailbox.model.ComposedMessageId;
 import org.apache.james.mailbox.model.Mailbox;
 import org.apache.james.mailbox.model.MailboxACL;
 import org.apache.james.mailbox.model.MailboxId;
 import org.apache.james.mailbox.model.MailboxPath;
 import org.apache.james.mailbox.model.MessageId;
-import org.apache.james.mailbox.model.MultimailboxesSearchQuery;
 import org.apache.james.mailbox.model.SearchQuery;
 import org.apache.james.mailbox.searchhighligt.SearchHighLighterContract;
 import org.apache.james.mailbox.searchhighligt.SearchHighlighter;
 import org.apache.james.mailbox.searchhighligt.SearchHighlighterConfiguration;
-import org.apache.james.mailbox.searchhighligt.SearchSnippet;
 import org.apache.james.mailbox.store.StoreMailboxManager;
 import org.apache.james.mailbox.store.StoreMessageManager;
 import org.apache.james.mailbox.store.extractor.JsoupTextExtractor;
 import org.apache.james.mailbox.store.search.MessageSearchIndex;
-import org.apache.james.mime4j.dom.Message;
-import org.apache.james.util.ClassLoaderUtils;
 import org.apache.lucene.store.ByteBuffersDirectory;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
 
 import com.github.fge.lambdas.Throwing;
 
-import reactor.core.publisher.Flux;
-
 class LuceneMemorySearchHighLighterTest implements SearchHighLighterContract {
     private MessageSearchIndex messageSearchIndex;
     private StoreMailboxManager storeMailboxManager;
@@ -127,45 +114,4 @@ class LuceneMemorySearchHighLighterTest implements SearchHighLighterContract {
             session(owner))).run();
     }
 
-
-    @Test
-    void shouldHighlightAttachmentTextContentWhenTextBodyDoesNotMatch() throws Exception {
-        assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.Attachment));
-        MailboxSession session = session(USERNAME1);
-
-        ComposedMessageId m1 = appendMessage(MessageManager.AppendCommand.from(
-                Message.Builder.of()
-                    .setTo("to@james.local")
-                    .setSubject("Hallo, Thx Matthieu for your help")
-                    .setBody("append contentA to inbox", StandardCharsets.UTF_8)),
-            session).getId();
-
-        // m2 has an attachment with text content: "This is a beautiful banana"
-        ComposedMessageId m2 = inboxMessageManager.appendMessage(
-            MessageManager.AppendCommand.builder()
-                .build(ClassLoaderUtils.getSystemResourceAsSharedStream("eml/emailWithTextAttachment.eml")),
-            session).getId();
-
-        verifyMessageWasIndexed(2);
-
-        String keywordSearch = "beautiful";
-        MultimailboxesSearchQuery multiMailboxSearch = MultimailboxesSearchQuery.from(SearchQuery.of(
-                new SearchQuery.ConjunctionCriterion(SearchQuery.Conjunction.OR,
-                    List.of(SearchQuery.bodyContains(keywordSearch),
-                        SearchQuery.attachmentContains(keywordSearch)))))
-            .inMailboxes(List.of(m1.getMailboxId(), m2.getMailboxId()))
-            .build();
-
-        List<SearchSnippet> searchSnippets = Flux.from(testee().highlightSearch(List.of(m1.getMessageId(), m2.getMessageId()), multiMailboxSearch, session))
-            .collectList()
-            .block();
-
-        assertThat(searchSnippets).hasSize(1);
-
-        assertThat(searchSnippets.getFirst().highlightedBody())
-            .isPresent()
-            .satisfies(highlightedBody -> {
-                assertThat(highlightedBody.get()).contains("This is a <mark>beautiful</mark> banana");
-            });
-    }
 }
diff --git a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/search/OpenSearchSearchHighlighterTest.java b/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/search/OpenSearchSearchHighlighterTest.java
index a0d73e19ba..ef4503a78d 100644
--- a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/search/OpenSearchSearchHighlighterTest.java
+++ b/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/search/OpenSearchSearchHighlighterTest.java
@@ -22,12 +22,9 @@ package org.apache.james.mailbox.opensearch.search;
 import static org.apache.james.mailbox.opensearch.search.OpenSearchSearcherTest.SEARCH_SIZE;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.awaitility.Durations.ONE_HUNDRED_MILLISECONDS;
-import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.time.ZoneId;
-import java.util.List;
 import java.util.UUID;
 
 import org.apache.james.backends.opensearch.DockerOpenSearchExtension;
@@ -37,18 +34,15 @@ import org.apache.james.backends.opensearch.ReactorOpenSearchClient;
 import org.apache.james.backends.opensearch.ReadAliasName;
 import org.apache.james.backends.opensearch.WriteAliasName;
 import org.apache.james.core.Username;
-import org.apache.james.mailbox.MailboxManager;
 import org.apache.james.mailbox.MailboxSession;
 import org.apache.james.mailbox.MessageManager;
 import org.apache.james.mailbox.inmemory.InMemoryMessageId;
 import org.apache.james.mailbox.inmemory.manager.InMemoryIntegrationResources;
-import org.apache.james.mailbox.model.ComposedMessageId;
 import org.apache.james.mailbox.model.Mailbox;
 import org.apache.james.mailbox.model.MailboxACL;
 import org.apache.james.mailbox.model.MailboxId;
 import org.apache.james.mailbox.model.MailboxPath;
 import org.apache.james.mailbox.model.MessageId;
-import org.apache.james.mailbox.model.MultimailboxesSearchQuery;
 import org.apache.james.mailbox.model.SearchQuery;
 import org.apache.james.mailbox.opensearch.IndexAttachments;
 import org.apache.james.mailbox.opensearch.IndexHeaders;
@@ -61,7 +55,6 @@ import org.apache.james.mailbox.opensearch.query.CriterionConverter;
 import org.apache.james.mailbox.opensearch.query.QueryConverter;
 import org.apache.james.mailbox.searchhighligt.SearchHighLighterContract;
 import org.apache.james.mailbox.searchhighligt.SearchHighlighter;
-import org.apache.james.mailbox.searchhighligt.SearchSnippet;
 import org.apache.james.mailbox.store.StoreMailboxManager;
 import org.apache.james.mailbox.store.StoreMessageManager;
 import org.apache.james.mailbox.store.search.MessageSearchIndex;
@@ -70,22 +63,17 @@ import org.apache.james.mailbox.tika.TikaExtension;
 import org.apache.james.mailbox.tika.TikaHttpClientImpl;
 import org.apache.james.mailbox.tika.TikaTextExtractor;
 import org.apache.james.metrics.tests.RecordingMetricFactory;
-import org.apache.james.mime4j.dom.Message;
-import org.apache.james.util.ClassLoaderUtils;
 import org.awaitility.Awaitility;
 import org.awaitility.Durations;
 import org.awaitility.core.ConditionFactory;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
 import com.github.fge.lambdas.Throwing;
 import com.google.common.collect.ImmutableSet;
 
-import reactor.core.publisher.Flux;
-
 public class OpenSearchSearchHighlighterTest implements SearchHighLighterContract {
     private MessageSearchIndex messageSearchIndex;
     private StoreMailboxManager storeMailboxManager;
@@ -205,40 +193,4 @@ public class OpenSearchSearchHighlighterTest implements SearchHighLighterContrac
                 .isEqualTo(indexedMessageCount));
     }
 
-    @Test
-    void shouldHighlightAttachmentTextContentWhenTextBodyDoesNotMatch() throws Exception {
-        assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.Attachment));
-        MailboxSession session = session(USERNAME1);
-
-        ComposedMessageId m1 = appendMessage(MessageManager.AppendCommand.from(
-                Message.Builder.of()
-                    .setTo("to@james.local")
-                    .setSubject("Hallo, Thx Matthieu for your help")
-                    .setBody("append contentA to inbox", StandardCharsets.UTF_8)),
-            session).getId();
-
-        // m2 has an attachment with text content: "This is a beautiful banana"
-        ComposedMessageId m2 = inboxMessageManager.appendMessage(
-            MessageManager.AppendCommand.builder()
-                .build(ClassLoaderUtils.getSystemResourceAsSharedStream("eml/emailWithTextAttachment.eml")),
-            session).getId();
-
-        verifyMessageWasIndexed(2);
-
-        String keywordSearch = "beautiful";
-        MultimailboxesSearchQuery multiMailboxSearch = MultimailboxesSearchQuery.from(SearchQuery.of(
-                new SearchQuery.ConjunctionCriterion(SearchQuery.Conjunction.OR,
-                    List.of(SearchQuery.bodyContains(keywordSearch),
-                        SearchQuery.attachmentContains(keywordSearch)))))
-            .inMailboxes(List.of(m1.getMailboxId(), m2.getMailboxId()))
-            .build();
-
-        List<SearchSnippet> searchSnippets = Flux.from(testee().highlightSearch(List.of(m1.getMessageId(), m2.getMessageId()), multiMailboxSearch, session))
-            .collectList()
-            .block();
-
-        assertThat(searchSnippets).hasSize(1);
-        assertThat(searchSnippets.getFirst().highlightedBody()).contains("This is a <mark>beautiful</mark> banana.");
-    }
-
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org


(james-project) 04/07: JAMES-4082 Lucene search - remove unused code & clean

Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git

commit fcc410b70ea833063152b070c5a300282dcbdd61
Author: TungTV <vt...@linagora.com>
AuthorDate: Mon Nov 4 08:29:24 2024 +0700

    JAMES-4082 Lucene search - remove unused code & clean
---
 .../lucene/search/LuceneMessageSearchIndex.java    | 525 ++-------------------
 .../lucene/search/LuceneSearchHighlighter.java     |   6 +-
 .../search/LuceneMailboxMessageFlagSearchTest.java |   2 +-
 3 files changed, 42 insertions(+), 491 deletions(-)

diff --git a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
index 81afadae54..385db3d0a2 100644
--- a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
+++ b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
@@ -20,10 +20,45 @@ package org.apache.james.mailbox.lucene.search;
 
 import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.ATTACHMENT_FILE_NAME_FIELD;
 import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.ATTACHMENT_TEXT_CONTENT_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.BASE_SUBJECT_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.BODY_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.FIRST_CC_MAILBOX_NAME_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.FIRST_FROM_MAILBOX_NAME_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.FIRST_TO_MAILBOX_NAME_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.FLAGS_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.HAS_ATTACHMENT_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.HEADERS_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.ID_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.INTERNAL_DATE_FIELD_DAY_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.INTERNAL_DATE_FIELD_HOUR_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.INTERNAL_DATE_FIELD_MINUTE_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.INTERNAL_DATE_FIELD_MONTH_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.INTERNAL_DATE_FIELD_SECOND_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.INTERNAL_DATE_FIELD_YEAR_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.MAILBOX_ID_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.MESSAGE_ID_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.MODSEQ_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.PREFIX_HEADER_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SAVE_DATE_FIELD_DAY_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SAVE_DATE_FIELD_HOUR_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SAVE_DATE_FIELD_MINUTE_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SAVE_DATE_FIELD_MONTH_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SAVE_DATE_FIELD_SECOND_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SAVE_DATE_FIELD_YEAR_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SENT_DATE_FIELD_DAY_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SENT_DATE_FIELD_HOUR_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SENT_DATE_FIELD_MINUTE_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SENT_DATE_FIELD_MONTH_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SENT_DATE_FIELD_SECOND_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SENT_DATE_FIELD_YEAR_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SENT_DATE_SORT_FIELD_MILLISECOND_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SIZE_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.THREAD_ID_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.UID_FIELD;
+import static org.apache.james.mailbox.lucene.search.LuceneIndexableDocument.createFlagsIdField;
 
 import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Collection;
@@ -41,7 +76,6 @@ import jakarta.inject.Inject;
 import jakarta.mail.Flags;
 import jakarta.mail.Flags.Flag;
 
-import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.james.mailbox.MailboxManager;
 import org.apache.james.mailbox.MailboxManager.SearchCapabilities;
@@ -53,7 +87,6 @@ import org.apache.james.mailbox.exception.UnsupportedSearchException;
 import org.apache.james.mailbox.extractor.TextExtractor;
 import org.apache.james.mailbox.model.Mailbox;
 import org.apache.james.mailbox.model.MailboxId;
-import org.apache.james.mailbox.model.MessageAttachmentMetadata;
 import org.apache.james.mailbox.model.MessageId;
 import org.apache.james.mailbox.model.MessageRange;
 import org.apache.james.mailbox.model.SearchQuery;
@@ -73,31 +106,10 @@ import org.apache.james.mailbox.store.MailboxSessionMapperFactory;
 import org.apache.james.mailbox.store.mail.model.MailboxMessage;
 import org.apache.james.mailbox.store.search.ListeningMessageSearchIndex;
 import org.apache.james.mailbox.store.search.SearchUtil;
-import org.apache.james.mime4j.MimeException;
-import org.apache.james.mime4j.dom.Header;
-import org.apache.james.mime4j.dom.address.Address;
-import org.apache.james.mime4j.dom.address.AddressList;
-import org.apache.james.mime4j.dom.address.Group;
-import org.apache.james.mime4j.dom.address.MailboxList;
-import org.apache.james.mime4j.dom.field.DateTimeField;
-import org.apache.james.mime4j.field.address.AddressFormatter;
-import org.apache.james.mime4j.field.address.LenientAddressParser;
-import org.apache.james.mime4j.message.SimpleContentHandler;
-import org.apache.james.mime4j.parser.MimeStreamParser;
-import org.apache.james.mime4j.stream.BodyDescriptor;
-import org.apache.james.mime4j.stream.MimeConfig;
-import org.apache.james.mime4j.util.MimeUtil;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.DateTools;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.LongPoint;
-import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.document.SortedSetDocValuesField;
-import org.apache.lucene.document.StoredField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexFormatTooOldException;
 import org.apache.lucene.index.IndexReader;
@@ -119,7 +131,6 @@ import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -160,214 +171,6 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
      */
     private static final int DEFAULT_MAX_QUERY_RESULTS = 100000;
 
-    /**
-     * {@link Field} which will contain the unique index of the {@link Document}
-     */
-    @VisibleForTesting
-    static final String ID_FIELD = "id";
-
-
-    /**
-     * {@link Field} which will contain uid of the {@link MailboxMessage}
-     */
-    static final String UID_FIELD = "uid";
-
-    /**
-     * {@link Field} boolean field that say if the message as an attachment or not
-     */
-    private static final String HAS_ATTACHMENT_FIELD = "hasAttachment";
-
-    /**
-     * {@link Field} which will contain the {@link Flags} of the {@link MailboxMessage}
-     */
-    static final String FLAGS_FIELD = "flags";
-
-    /**
-     * {@link Field} which will contain the size of the {@link MailboxMessage}
-     */
-    private static final String SIZE_FIELD = "size";
-
-    /**
-     * {@link Field} which will contain the body of the {@link MailboxMessage}
-     */
-    static final String BODY_FIELD = "body";
-
-    /**
-     * Prefix which will be used for each message header to store it also in a seperate {@link Field}
-     */
-    private static final String PREFIX_HEADER_FIELD = "header_";
-
-    /**
-     * {@link Field} which will contain the whole message header of the {@link MailboxMessage}
-     */
-    private static final String HEADERS_FIELD = "headers";
-
-    /**
-     * {@link Field} which will contain the mod-sequence of the message
-     */
-    private static final String MODSEQ_FIELD = "modSeq";
-
-    /**
-     * {@link Field} which will contain the threadId of the message
-     */
-    private static final String THREAD_ID_FIELD = "threadId";
-
-    /**
-     * {@link Field} which will contain the TO-Address of the message
-     */
-    private static final String TO_FIELD = "to";
-
-    private static final String FIRST_TO_MAILBOX_NAME_FIELD = "firstToMailboxName";
-    private static final String FIRST_TO_MAILBOX_DISPLAY_FIELD = "firstToMailboxDisplay";
-
-    /**
-     * {@link Field} which will contain the CC-Address of the message
-     */
-    private static final String CC_FIELD = "cc";
-
-    private static final String FIRST_CC_MAILBOX_NAME_FIELD = "firstCcMailboxName";
-
-
-    /**
-     * {@link Field} which will contain the FROM-Address of the message
-     */
-    private static final String FROM_FIELD = "from";
-
-    private static final String FIRST_FROM_MAILBOX_NAME_FIELD = "firstFromMailboxName";
-    private static final String FIRST_FROM_MAILBOX_DISPLAY_FIELD = "firstFromMailboxDisplay";
-
-    /**
-     * {@link Field} which will contain the BCC-Address of the message
-     */
-    private static final String BCC_FIELD = "bcc";
-
-
-    static final String BASE_SUBJECT_FIELD = "baseSubject";
-    static final String SUBJECT_FIELD = "subject";
-
-    /**
-     * {@link Field} which contain the internalDate of the message with YEAR-Resolution
-     */
-    private static final String INTERNAL_DATE_FIELD_YEAR_RESOLUTION = "internaldateYearResolution";
-
-
-    /**
-     * {@link Field} which contain the internalDate of the message with MONTH-Resolution
-     */
-    private static final String INTERNAL_DATE_FIELD_MONTH_RESOLUTION = "internaldateMonthResolution";
-
-    /**
-     * {@link Field} which contain the internalDate of the message with DAY-Resolution
-     */
-    private static final String INTERNAL_DATE_FIELD_DAY_RESOLUTION = "internaldateDayResolution";
-
-    /**
-     * {@link Field} which contain the internalDate of the message with HOUR-Resolution
-     */
-    private static final String INTERNAL_DATE_FIELD_HOUR_RESOLUTION = "internaldateHourResolution";
-
-    /**
-     * {@link Field} which contain the internalDate of the message with MINUTE-Resolution
-     */
-    private static final String INTERNAL_DATE_FIELD_MINUTE_RESOLUTION = "internaldateMinuteResolution";
-
-    /**
-     * {@link Field} which contain the internalDate of the message with SECOND-Resolution
-     */
-    private static final String INTERNAL_DATE_FIELD_SECOND_RESOLUTION = "internaldateSecondResolution";
-
-
-    /**
-     * {@link Field} which contain the internalDate of the message with MILLISECOND-Resolution
-     */
-    private static final String INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION = "internaldateMillisecondResolution";
-
-    /**
-     * {@link Field} which contain the saveDate of the message with YEAR-Resolution
-     */
-    private static final String SAVE_DATE_FIELD_YEAR_RESOLUTION = "saveDateYearResolution";
-
-    /**
-     * {@link Field} which contain the saveDate of the message with MONTH-Resolution
-     */
-    private static final String SAVE_DATE_FIELD_MONTH_RESOLUTION = "saveDateMonthResolution";
-
-    /**
-     * {@link Field} which contain the saveDate of the message with DAY-Resolution
-     */
-    private static final String SAVE_DATE_FIELD_DAY_RESOLUTION = "saveDateDayResolution";
-
-    /**
-     * {@link Field} which contain the saveDate of the message with HOUR-Resolution
-     */
-    private static final String SAVE_DATE_FIELD_HOUR_RESOLUTION = "saveDateHourResolution";
-
-    /**
-     * {@link Field} which contain the saveDate of the message with MINUTE-Resolution
-     */
-    private static final String SAVE_DATE_FIELD_MINUTE_RESOLUTION = "saveDateMinuteResolution";
-
-    /**
-     * {@link Field} which contain the saveDate of the message with SECOND-Resolution
-     */
-    private static final String SAVE_DATE_FIELD_SECOND_RESOLUTION = "saveDateSecondResolution";
-
-    /**
-     * {@link Field} which will contain the id of the {@link Mailbox}
-     */
-    static final String MAILBOX_ID_FIELD = "mailboxid";
-
-    /**
-     * {@link Field} which will contain the user of the {@link MailboxSession}
-     */
-    private static final String USERS = "userSession";
-    /**
-     * {@link Field} which will contain the id of the {@link MessageId}
-     */
-    static final String MESSAGE_ID_FIELD = "messageid";
-
-    /**
-     * {@link Field} which contain the Date header of the message with YEAR-Resolution
-     */
-    private static final String SENT_DATE_FIELD_YEAR_RESOLUTION = "sentdateYearResolution";
-
-
-    /**
-     * {@link Field} which contain the Date header of the message with MONTH-Resolution
-     */
-    private static final String SENT_DATE_FIELD_MONTH_RESOLUTION = "sentdateMonthResolution";
-
-    /**
-     * {@link Field} which contain the Date header of the message with DAY-Resolution
-     */
-    private static final String SENT_DATE_FIELD_DAY_RESOLUTION = "sentdateDayResolution";
-
-    /**
-     * {@link Field} which contain the Date header of the message with HOUR-Resolution
-     */
-    private static final String SENT_DATE_FIELD_HOUR_RESOLUTION = "sentdateHourResolution";
-
-    /**
-     * {@link Field} which contain the Date header of the message with MINUTE-Resolution
-     */
-    private static final String SENT_DATE_FIELD_MINUTE_RESOLUTION = "sentdateMinuteResolution";
-
-    /**
-     * {@link Field} which contain the Date header of the message with SECOND-Resolution
-     */
-    private static final String SENT_DATE_FIELD_SECOND_RESOLUTION = "sentdateSecondResolution";
-
-
-    /**
-     * {@link Field} which contain the Date header of the message with MILLISECOND-Resolution
-     */
-    private static final String SENT_DATE_FIELD_MILLISECOND_RESOLUTION = "sentdateMillisecondResolution";
-
-    private static final String SENT_DATE_SORT_FIELD_MILLISECOND_RESOLUTION = "sentdateSort";
-
-    private static final String MEDIA_TYPE_TEXT = "text";
-    private static final String MEDIA_TYPE_MESSAGE = "message";
-    private static final String DEFAULT_ENCODING = "US-ASCII";
     private static final boolean INCLUDE_LOWER = true;
     private static final boolean INCLUDE_UPPER = true;
 
@@ -564,214 +367,6 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
         return queryBuilder.build();
     }
 
-    /**
-     * Create a new {@link Document} for the given {@link MailboxMessage}. This Document does not contain any flags data. The {@link Flags} are stored in a seperate Document.
-     * <p>
-     * See {@link #createFlagsDocument(MailboxMessage)}
-     */
-    private Document createMessageDocument(final MailboxSession session, final MailboxMessage membership) throws IOException, MimeException {
-        final Document doc = new Document();
-        // TODO: Better handling
-        doc.add(new StringField(USERS, session.getUser().asString().toUpperCase(Locale.US), Store.YES));
-        doc.add(new StringField(MAILBOX_ID_FIELD, membership.getMailboxId().serialize().toUpperCase(Locale.US), Store.YES));
-        doc.add(new NumericDocValuesField(UID_FIELD, membership.getUid().asLong()));
-        doc.add(new LongPoint(UID_FIELD, membership.getUid().asLong()));
-        doc.add(new StoredField(UID_FIELD, membership.getUid().asLong()));
-        doc.add(new StringField(HAS_ATTACHMENT_FIELD, Boolean.toString(hasAttachment(membership)), Store.YES));
-
-        String serializedMessageId = SearchUtil.getSerializedMessageIdIfSupportedByUnderlyingStorageOrNull(membership);
-        if (serializedMessageId != null) {
-            doc.add(new StringField(MESSAGE_ID_FIELD, serializedMessageId, Store.YES));
-        }
-        String serializedThreadId = SearchUtil.getSerializedThreadIdIfSupportedByUnderlyingStorageOrNull(membership);
-        if (serializedThreadId != null) {
-            doc.add(new StringField(THREAD_ID_FIELD, serializedThreadId, Store.YES));
-        }
-
-        // create a unique key for the document which can be used later on updates to find the document
-        doc.add(new StringField(ID_FIELD, membership.getMailboxId().serialize().toUpperCase(Locale.US) + "-" + membership.getUid().asLong(), Store.YES));
-
-        doc.add(new StringField(INTERNAL_DATE_FIELD_YEAR_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.YEAR), Store.NO));
-        doc.add(new StringField(INTERNAL_DATE_FIELD_MONTH_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.MONTH), Store.NO));
-        doc.add(new StringField(INTERNAL_DATE_FIELD_DAY_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.DAY), Store.NO));
-        doc.add(new StringField(INTERNAL_DATE_FIELD_HOUR_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.HOUR), Store.NO));
-        doc.add(new StringField(INTERNAL_DATE_FIELD_MINUTE_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.MINUTE), Store.NO));
-        doc.add(new StringField(INTERNAL_DATE_FIELD_SECOND_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.SECOND), Store.NO));
-        doc.add(new NumericDocValuesField(INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION, Long.parseLong(DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.MILLISECOND))));
-
-        membership.getSaveDate().ifPresent(saveDate -> {
-            doc.add(new StringField(SAVE_DATE_FIELD_YEAR_RESOLUTION, DateTools.dateToString(saveDate, DateTools.Resolution.YEAR), Store.NO));
-            doc.add(new StringField(SAVE_DATE_FIELD_MONTH_RESOLUTION, DateTools.dateToString(saveDate, DateTools.Resolution.MONTH), Store.NO));
-            doc.add(new StringField(SAVE_DATE_FIELD_DAY_RESOLUTION, DateTools.dateToString(saveDate, DateTools.Resolution.DAY), Store.NO));
-            doc.add(new StringField(SAVE_DATE_FIELD_HOUR_RESOLUTION, DateTools.dateToString(saveDate, DateTools.Resolution.HOUR), Store.NO));
-            doc.add(new StringField(SAVE_DATE_FIELD_MINUTE_RESOLUTION, DateTools.dateToString(saveDate, DateTools.Resolution.MINUTE), Store.NO));
-            doc.add(new StringField(SAVE_DATE_FIELD_SECOND_RESOLUTION, DateTools.dateToString(saveDate, DateTools.Resolution.SECOND), Store.NO));
-        });
-
-        doc.add(new LongPoint(SIZE_FIELD, membership.getFullContentOctets()));
-        doc.add(new NumericDocValuesField(SIZE_FIELD, membership.getFullContentOctets()));
-
-        // content handler which will index the headers and the body of the message
-        SimpleContentHandler handler = new SimpleContentHandler() {
-
-
-            @Override
-            public void headers(Header header) {
-
-                Date sentDate = null;
-                String firstFromMailbox = "";
-                String firstToMailbox = "";
-                String firstCcMailbox = "";
-                String firstFromDisplay = "";
-                String firstToDisplay = "";
-
-                for (org.apache.james.mime4j.stream.Field f : header) {
-                    String headerName = f.getName().toUpperCase(Locale.US);
-                    String headerValue = f.getBody().toUpperCase(Locale.US);
-                    String fullValue = f.toString().toUpperCase(Locale.US);
-                    doc.add(new TextField(HEADERS_FIELD, fullValue, Store.NO));
-                    doc.add(new TextField(PREFIX_HEADER_FIELD + headerName, headerValue, Store.NO));
-
-                    if (f instanceof DateTimeField dateTimeField) {
-                        sentDate = dateTimeField.getDate();
-                    }
-                    String field = null;
-                    if ("To".equalsIgnoreCase(headerName)) {
-                        field = TO_FIELD;
-                    } else if ("From".equalsIgnoreCase(headerName)) {
-                        field = FROM_FIELD;
-                    } else if ("Cc".equalsIgnoreCase(headerName)) {
-                        field = CC_FIELD;
-                    } else if ("Bcc".equalsIgnoreCase(headerName)) {
-                        field = BCC_FIELD;
-                    }
-
-
-                    // Check if we can index the address in the right manner
-                    if (field != null) {
-                        // not sure if we really should reparse it. It maybe be better to check just for the right type.
-                        // But this impl was easier in the first place
-                        AddressList aList = LenientAddressParser.DEFAULT.parseAddressList(MimeUtil.unfold(f.getBody()));
-                        for (int i = 0; i < aList.size(); i++) {
-                            Address address = aList.get(i);
-                            if (address instanceof org.apache.james.mime4j.dom.address.Mailbox mailbox) {
-                                String value = AddressFormatter.DEFAULT.encode(mailbox).toUpperCase(Locale.US);
-                                doc.add(new TextField(field, value, Store.NO));
-                                if (i == 0) {
-                                    String mailboxAddress = SearchUtil.getMailboxAddress(mailbox);
-                                    String mailboxDisplay = SearchUtil.getDisplayAddress(mailbox);
-
-                                    if ("To".equalsIgnoreCase(headerName)) {
-                                        firstToMailbox = mailboxAddress;
-                                        firstToDisplay = mailboxDisplay;
-                                    } else if ("From".equalsIgnoreCase(headerName)) {
-                                        firstFromMailbox = mailboxAddress;
-                                        firstFromDisplay = mailboxDisplay;
-
-                                    } else if ("Cc".equalsIgnoreCase(headerName)) {
-                                        firstCcMailbox = mailboxAddress;
-                                    }
-
-                                }
-                            } else if (address instanceof Group) {
-                                MailboxList mList = ((Group) address).getMailboxes();
-                                for (int a = 0; a < mList.size(); a++) {
-                                    org.apache.james.mime4j.dom.address.Mailbox mailbox = mList.get(a);
-                                    String value = AddressFormatter.DEFAULT.encode(mailbox).toUpperCase(Locale.US);
-                                    doc.add(new TextField(field, value, Store.NO));
-
-                                    if (i == 0 && a == 0) {
-                                        String mailboxAddress = SearchUtil.getMailboxAddress(mailbox);
-                                        String mailboxDisplay = SearchUtil.getDisplayAddress(mailbox);
-
-                                        if ("To".equalsIgnoreCase(headerName)) {
-                                            firstToMailbox = mailboxAddress;
-                                            firstToDisplay = mailboxDisplay;
-                                        } else if ("From".equalsIgnoreCase(headerName)) {
-                                            firstFromMailbox = mailboxAddress;
-                                            firstFromDisplay = mailboxDisplay;
-
-                                        } else if ("Cc".equalsIgnoreCase(headerName)) {
-                                            firstCcMailbox = mailboxAddress;
-                                        }
-                                    }
-                                }
-                            }
-                        }
-
-                        doc.add(new TextField(field, headerValue, Store.NO));
-
-                    } else if (headerName.equalsIgnoreCase("Subject")) {
-                        doc.add(new StringField(SUBJECT_FIELD, f.getBody(), Store.YES));
-                        doc.add(new StringField(BASE_SUBJECT_FIELD, SearchUtil.getBaseSubject(headerValue), Store.YES));
-                        doc.add(new SortedSetDocValuesField(BASE_SUBJECT_FIELD, new BytesRef(SearchUtil.getBaseSubject(headerValue))));
-                    }
-                }
-                if (sentDate == null) {
-                    sentDate = membership.getInternalDate();
-                } else {
-
-                    doc.add(new StringField(SENT_DATE_FIELD_YEAR_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.YEAR), Store.NO));
-                    doc.add(new StringField(SENT_DATE_FIELD_MONTH_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.MONTH), Store.NO));
-                    doc.add(new StringField(SENT_DATE_FIELD_DAY_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.DAY), Store.NO));
-                    doc.add(new StringField(SENT_DATE_FIELD_HOUR_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.HOUR), Store.NO));
-                    doc.add(new StringField(SENT_DATE_FIELD_MINUTE_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.MINUTE), Store.NO));
-                    doc.add(new StringField(SENT_DATE_FIELD_SECOND_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.SECOND), Store.NO));
-                    doc.add(new StringField(SENT_DATE_FIELD_MILLISECOND_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.MILLISECOND), Store.NO));
-
-                }
-                // Remove existing SENT_DATE_SORT_FIELD field if it exists
-                doc.removeField(SENT_DATE_SORT_FIELD_MILLISECOND_RESOLUTION);
-                doc.add(new NumericDocValuesField(SENT_DATE_SORT_FIELD_MILLISECOND_RESOLUTION, Long.parseLong(DateTools.dateToString(sentDate, DateTools.Resolution.MILLISECOND))));
-
-                doc.add(new StringField(FIRST_FROM_MAILBOX_NAME_FIELD, firstFromMailbox, Store.YES));
-                doc.add(new SortedSetDocValuesField(FIRST_FROM_MAILBOX_NAME_FIELD, new BytesRef(firstFromMailbox)));
-                doc.add(new StringField(FIRST_TO_MAILBOX_NAME_FIELD, firstToMailbox, Store.YES));
-                doc.add(new SortedSetDocValuesField(FIRST_TO_MAILBOX_NAME_FIELD, new BytesRef(firstToMailbox)));
-                doc.add(new StringField(FIRST_CC_MAILBOX_NAME_FIELD, firstCcMailbox, Store.YES));
-                doc.add(new SortedSetDocValuesField(FIRST_CC_MAILBOX_NAME_FIELD, new BytesRef(firstCcMailbox)));
-                doc.add(new StringField(FIRST_FROM_MAILBOX_DISPLAY_FIELD, firstFromDisplay, Store.YES));
-                doc.add(new StringField(FIRST_TO_MAILBOX_DISPLAY_FIELD, firstToDisplay, Store.YES));
-
-            }
-
-            @Override
-            public void body(BodyDescriptor desc, InputStream in) throws IOException {
-                String mediaType = desc.getMediaType();
-                if (MEDIA_TYPE_TEXT.equalsIgnoreCase(mediaType) || MEDIA_TYPE_MESSAGE.equalsIgnoreCase(mediaType)) {
-                    String cset = desc.getCharset();
-                    if (cset == null) {
-                        cset = DEFAULT_ENCODING;
-                    }
-                    Charset charset;
-                    try {
-                        charset = Charset.forName(cset);
-                    } catch (Exception e) {
-                        // Invalid charset found so fallback toe the DEFAULT_ENCODING
-                        charset = Charset.forName(DEFAULT_ENCODING);
-                    }
-
-                    String bodyContent = IOUtils.toString(in, charset);
-                    doc.add(new TextField(BODY_FIELD, bodyContent, Store.YES));
-                }
-            }
-
-        };
-        //config.setStrictParsing(false);
-        MimeStreamParser parser = new MimeStreamParser(MimeConfig.PERMISSIVE);
-        parser.setContentDecoding(true);
-        parser.setContentHandler(handler);
-
-        // parse the message to index headers and body
-        parser.parse(membership.getFullContent());
-
-        return doc;
-    }
-
-    private static boolean hasAttachment(MailboxMessage membership) {
-       return MessageAttachmentMetadata.hasNonInlinedAttachment(membership.getAttachments());
-    }
-
     private String toSentDateField(DateResolution res) {
         return switch (res) {
             case Year -> SENT_DATE_FIELD_YEAR_RESOLUTION;
@@ -1175,56 +770,12 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
     private void update(MailboxId mailboxId, MessageUid uid, Flags f) throws IOException {
         var flagsID = createFlagsIdField(mailboxId, uid);
         var term = new Term(ID_FIELD, flagsID);
-        var doc = createFlagsDocument(mailboxId, uid, f);
+        var doc = indexableDocument.createFlagsDocument(mailboxId, uid, f);
         log.trace("Updating flags document, mailboxId:{}, message uid: {}, flags:'{}', term: {}, new document: {}",
                 mailboxId, uid, f, term, doc);
         writer.updateDocument(term, doc);
     }
 
-    /**
-     * Index the {@link Flags} and add it to the {@link Document}
-     */
-    private Document createFlagsDocument(MailboxMessage message) {
-        return createFlagsDocument(message.getMailboxId(), message.getUid(), message.createFlags());
-    }
-
-    private Document createFlagsDocument(MailboxId mailboxId, final MessageUid messageUid, Flags flags) {
-        Document doc = new Document();
-        doc.add(new StringField(ID_FIELD, createFlagsIdField(mailboxId, messageUid), Store.YES));
-        doc.add(new StringField(MAILBOX_ID_FIELD, mailboxId.serialize(), Store.YES));
-
-        doc.add(new NumericDocValuesField(UID_FIELD, messageUid.asLong()));
-        doc.add(new LongPoint(UID_FIELD, messageUid.asLong()));
-        doc.add(new StoredField(UID_FIELD, messageUid.asLong()));
-
-        indexFlags(doc, flags);
-        return doc;
-    }
-
-    private static String createFlagsIdField(MailboxId mailboxId, MessageUid messageUid) {
-        return "flags-" + mailboxId.serialize() + "-" + messageUid.asLong();
-    }
-
-    /**
-     * Add the given {@link Flags} to the {@link Document}
-     */
-    private void indexFlags(Document doc, Flags f) {
-        Flag[] flags = f.getSystemFlags();
-        for (Flag flag : flags) {
-            doc.add(new StringField(FLAGS_FIELD, toString(flag), Store.YES));
-        }
-
-        String[] userFlags = f.getUserFlags();
-        for (String userFlag : userFlags) {
-            doc.add(new StringField(FLAGS_FIELD, userFlag, Store.YES));
-        }
-
-        // if no flags are there we just use a empty field
-        if (flags.length == 0 && userFlags.length == 0) {
-            doc.add(new StringField(FLAGS_FIELD, "",Store.NO));
-        }
-    }
-
     private Query createQuery(MessageRange range) {
         return switch (range.getType()) {
             case ONE -> LongPoint.newRangeQuery(UID_FIELD,
diff --git a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneSearchHighlighter.java b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneSearchHighlighter.java
index 7a6804988d..830108f4b6 100644
--- a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneSearchHighlighter.java
+++ b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneSearchHighlighter.java
@@ -20,9 +20,9 @@
 package org.apache.james.mailbox.lucene.search;
 
 import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.ATTACHMENT_TEXT_CONTENT_FIELD;
-import static org.apache.james.mailbox.lucene.search.LuceneMessageSearchIndex.BODY_FIELD;
-import static org.apache.james.mailbox.lucene.search.LuceneMessageSearchIndex.MESSAGE_ID_FIELD;
-import static org.apache.james.mailbox.lucene.search.LuceneMessageSearchIndex.SUBJECT_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.BODY_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.MESSAGE_ID_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SUBJECT_FIELD;
 
 import java.io.IOException;
 import java.util.Arrays;
diff --git a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageFlagSearchTest.java b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageFlagSearchTest.java
index eca27dae24..dcc4182564 100644
--- a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageFlagSearchTest.java
+++ b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageFlagSearchTest.java
@@ -18,7 +18,7 @@
  ****************************************************************/
 package org.apache.james.mailbox.lucene.search;
 
-import static org.apache.james.mailbox.lucene.search.LuceneMessageSearchIndex.ID_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.ID_FIELD;
 import static org.apache.james.mailbox.lucene.search.LuceneTestsUtils.documentStringFormatter;
 import static org.apache.james.mailbox.lucene.search.LuceneTestsUtils.getAllDocumentsFromRepository;
 import static org.assertj.core.api.Assertions.assertThat;


---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org


(james-project) 01/07: JAMES-4082 Move MimePartParser and related classes from the `opensearch` module to the `mailbox-store` module

Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git

commit b4e7f4cda41f77d9f19bfd3334820a2ee8da2435
Author: TungTV <vt...@linagora.com>
AuthorDate: Mon Nov 4 08:29:24 2024 +0700

    JAMES-4082 Move MimePartParser and related classes from the `opensearch` module to the `mailbox-store` module
    
    - This change facilitates easier reuse of these classes in other modules.
---
 .../{SerializableMessage.java => EMailerDto.java}  |  8 +++-
 .../mailbox/opensearch/json/EMailersDto.java}      | 24 ++++------
 .../{SerializableMessage.java => HeaderDto.java}   | 11 ++++-
 .../mailbox/opensearch/json/IndexableMessage.java  | 52 +++++++++++++---------
 .../mailbox/opensearch/json/MimePartDto.java}      | 31 ++++++-------
 .../{SerializableMessage.java => SubjectsDto.java} |  6 ++-
 .../opensearch/query/CriterionConverter.java       |  2 +-
 .../opensearch/json/IndexableMessageTest.java      |  2 +-
 .../james/mailbox/store/search/mime}/EMailer.java  |  6 +--
 .../james/mailbox/store/search/mime}/EMailers.java |  4 +-
 .../store/search/mime}/HeaderCollection.java       |  5 +--
 .../james/mailbox/store/search/mime}/MimePart.java | 15 +------
 .../search/mime}/MimePartContainerBuilder.java     |  2 +-
 .../mailbox/store/search/mime}/MimePartParser.java |  2 +-
 .../search/mime}/RootMimePartContainerBuilder.java |  2 +-
 .../store/search/mime}/SerializableMessage.java    |  2 +-
 .../james/mailbox/store/search/mime}/Subjects.java |  4 +-
 .../mailbox/store/search/mime}/EMailerTest.java    |  2 +-
 .../mailbox/store/search/mime}/EMailersTest.java   |  2 +-
 .../mailbox/store/search/mime}/FieldImpl.java      |  2 +-
 .../store/search/mime}/HeaderCollectionTest.java   |  2 +-
 .../mailbox/store/search/mime}/MimePartTest.java   |  2 +-
 .../mailbox/store/search/mime}/SubjectsTest.java   |  2 +-
 23 files changed, 92 insertions(+), 98 deletions(-)

diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/SerializableMessage.java b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/EMailerDto.java
similarity index 77%
copy from mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/SerializableMessage.java
copy to mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/EMailerDto.java
index e91a295066..28e30cdb69 100644
--- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/SerializableMessage.java
+++ b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/EMailerDto.java
@@ -19,7 +19,11 @@
 
 package org.apache.james.mailbox.opensearch.json;
 
-public interface SerializableMessage {
+import java.util.Optional;
 
-    String serialize();
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public record EMailerDto(@JsonProperty(JsonMessageConstants.EMailer.NAME) Optional<String> name,
+                         @JsonProperty(JsonMessageConstants.EMailer.ADDRESS) String address,
+                         @JsonProperty(JsonMessageConstants.EMailer.DOMAIN) String domain) {
 }
diff --git a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/EMailerTest.java b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/EMailersDto.java
similarity index 71%
copy from mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/EMailerTest.java
copy to mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/EMailersDto.java
index f3790ae555..3936dab398 100644
--- a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/EMailerTest.java
+++ b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/EMailersDto.java
@@ -19,24 +19,18 @@
 
 package org.apache.james.mailbox.opensearch.json;
 
-import static org.assertj.core.api.Assertions.assertThatCode;
+import java.util.Set;
+import java.util.stream.Collectors;
 
-import java.util.Optional;
+import org.apache.james.mailbox.store.search.mime.EMailers;
 
-import org.junit.jupiter.api.Test;
+import com.fasterxml.jackson.annotation.JsonValue;
 
-import nl.jqno.equalsverifier.EqualsVerifier;
+public record EMailersDto(@JsonValue Set<EMailerDto> emailers) {
 
-class EMailerTest {
-    @Test
-    void eMailerShouldRespectBeanContract() {
-        EqualsVerifier.forClass(EMailer.class)
-            .verify();
-    }
-
-    @Test
-    void shouldSupportNullDomain() {
-        assertThatCode(() -> new EMailer(Optional.empty(), "localPart", null))
-            .doesNotThrowAnyException();
+    public static EMailersDto from(EMailers eMailers) {
+        return new EMailersDto(eMailers.getEmailers().stream()
+            .map(emailer -> new EMailerDto(emailer.getName(), emailer.getAddress(), emailer.getDomain()))
+            .collect(Collectors.toSet()));
     }
 }
diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/SerializableMessage.java b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/HeaderDto.java
similarity index 73%
copy from mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/SerializableMessage.java
copy to mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/HeaderDto.java
index e91a295066..4fc494245e 100644
--- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/SerializableMessage.java
+++ b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/HeaderDto.java
@@ -19,7 +19,14 @@
 
 package org.apache.james.mailbox.opensearch.json;
 
-public interface SerializableMessage {
+import org.apache.james.mailbox.store.search.mime.HeaderCollection;
 
-    String serialize();
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public record HeaderDto(@JsonProperty(JsonMessageConstants.HEADER.NAME) String headerName,
+                        @JsonProperty(JsonMessageConstants.HEADER.VALUE) String value) {
+
+    public static HeaderDto from(HeaderCollection.Header header) {
+        return new HeaderDto(header.getHeaderName(), header.getValue());
+    }
 }
diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/IndexableMessage.java b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/IndexableMessage.java
index 23569aed72..722d8b8988 100644
--- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/IndexableMessage.java
+++ b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/IndexableMessage.java
@@ -27,6 +27,7 @@ import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.List;
 import java.util.Optional;
+import java.util.stream.Collectors;
 
 import org.apache.james.mailbox.ModSeq;
 import org.apache.james.mailbox.extractor.TextExtractor;
@@ -36,6 +37,11 @@ import org.apache.james.mailbox.opensearch.IndexBody;
 import org.apache.james.mailbox.opensearch.IndexHeaders;
 import org.apache.james.mailbox.store.mail.model.MailboxMessage;
 import org.apache.james.mailbox.store.search.SearchUtil;
+import org.apache.james.mailbox.store.search.mime.EMailers;
+import org.apache.james.mailbox.store.search.mime.HeaderCollection;
+import org.apache.james.mailbox.store.search.mime.MimePart;
+import org.apache.james.mailbox.store.search.mime.MimePartParser;
+import org.apache.james.mailbox.store.search.mime.Subjects;
 import org.apache.james.mime4j.MimeException;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
@@ -226,15 +232,15 @@ public class IndexableMessage {
         return new Builder();
     }
 
-    private final List<MimePart> attachments;
-    private final EMailers bcc;
+    private final List<MimePartDto> attachments;
+    private final EMailersDto bcc;
     private final Optional<String> bodyHtml;
     private final Optional<String> bodyText;
-    private final EMailers cc;
+    private final EMailersDto cc;
     private final String date;
-    private final EMailers from;
+    private final EMailersDto from;
     private final boolean hasAttachment;
-    private final List<HeaderCollection.Header> headers;
+    private final List<HeaderDto> headers;
     private final boolean isAnswered;
     private final boolean isDeleted;
     private final boolean isDraft;
@@ -250,9 +256,9 @@ public class IndexableMessage {
     private final String sentDate;
     private final Optional<String> saveDate;
     private final long size;
-    private final Subjects subjects;
+    private final SubjectsDto subjects;
     private final String subType;
-    private final EMailers to;
+    private final EMailersDto to;
     private final long uid;
     private final String[] userFlags;
     private final Optional<String> mimeMessageID;
@@ -284,15 +290,19 @@ public class IndexableMessage {
                              long uid,
                              String[] userFlags,
                              Optional<String> mimeMessageID) {
-        this.attachments = attachments;
-        this.bcc = bcc;
+        this.attachments = attachments.stream()
+            .map(MimePartDto::from)
+            .toList();
+        this.bcc = EMailersDto.from(bcc);
         this.bodyHtml = bodyHtml;
         this.bodyText = bodyText;
-        this.cc = cc;
+        this.cc = EMailersDto.from(cc);
         this.date = date;
-        this.from = from;
+        this.from = EMailersDto.from(from);
         this.hasAttachment = hasAttachment;
-        this.headers = headers;
+        this.headers = headers.stream()
+            .map(HeaderDto::from)
+            .collect(Collectors.toList());
         this.isAnswered = isAnswered;
         this.isDeleted = isDeleted;
         this.isDraft = isDraft;
@@ -308,21 +318,21 @@ public class IndexableMessage {
         this.sentDate = sentDate;
         this.saveDate = saveDate;
         this.size = size;
-        this.subjects = subjects;
+        this.subjects = new SubjectsDto(subjects.getSubjects());
         this.subType = subType;
-        this.to = to;
+        this.to = EMailersDto.from(to);
         this.uid = uid;
         this.userFlags = userFlags;
         this.mimeMessageID = mimeMessageID;
     }
 
     @JsonProperty(JsonMessageConstants.ATTACHMENTS)
-    public List<MimePart> getAttachments() {
+    public List<MimePartDto> getAttachments() {
         return attachments;
     }
 
     @JsonProperty(JsonMessageConstants.BCC)
-    public EMailers getBcc() {
+    public EMailersDto getBcc() {
         return bcc;
     }
 
@@ -337,7 +347,7 @@ public class IndexableMessage {
     }
 
     @JsonProperty(JsonMessageConstants.CC)
-    public EMailers getCc() {
+    public EMailersDto getCc() {
         return cc;
     }
 
@@ -347,7 +357,7 @@ public class IndexableMessage {
     }
 
     @JsonProperty(JsonMessageConstants.FROM)
-    public EMailers getFrom() {
+    public EMailersDto getFrom() {
         return from;
     }
 
@@ -357,7 +367,7 @@ public class IndexableMessage {
     }
 
     @JsonProperty(JsonMessageConstants.HEADERS)
-    public List<HeaderCollection.Header> getHeaders() {
+    public List<HeaderDto> getHeaders() {
         return headers;
     }
 
@@ -402,7 +412,7 @@ public class IndexableMessage {
     }
 
     @JsonProperty(JsonMessageConstants.SUBJECT)
-    public Subjects getSubjects() {
+    public SubjectsDto getSubjects() {
         return subjects;
     }
 
@@ -412,7 +422,7 @@ public class IndexableMessage {
     }
 
     @JsonProperty(JsonMessageConstants.TO)
-    public EMailers getTo() {
+    public EMailersDto getTo() {
         return to;
     }
 
diff --git a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/EMailerTest.java b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MimePartDto.java
similarity index 53%
copy from mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/EMailerTest.java
copy to mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MimePartDto.java
index f3790ae555..d85a2a5a07 100644
--- a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/EMailerTest.java
+++ b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MimePartDto.java
@@ -19,24 +19,25 @@
 
 package org.apache.james.mailbox.opensearch.json;
 
-import static org.assertj.core.api.Assertions.assertThatCode;
-
 import java.util.Optional;
 
-import org.junit.jupiter.api.Test;
-
-import nl.jqno.equalsverifier.EqualsVerifier;
+import org.apache.james.mailbox.store.search.mime.MimePart;
 
-class EMailerTest {
-    @Test
-    void eMailerShouldRespectBeanContract() {
-        EqualsVerifier.forClass(EMailer.class)
-            .verify();
-    }
+import com.fasterxml.jackson.annotation.JsonProperty;
 
-    @Test
-    void shouldSupportNullDomain() {
-        assertThatCode(() -> new EMailer(Optional.empty(), "localPart", null))
-            .doesNotThrowAnyException();
+public record MimePartDto(@JsonProperty(JsonMessageConstants.Attachment.TEXT_CONTENT) Optional<String> bodyTextContent,
+                          @JsonProperty(JsonMessageConstants.Attachment.MEDIA_TYPE) Optional<String> mediaType,
+                          @JsonProperty(JsonMessageConstants.Attachment.SUBTYPE) Optional<String> subType,
+                          @JsonProperty(JsonMessageConstants.Attachment.FILENAME) Optional<String> fileName,
+                          @JsonProperty(JsonMessageConstants.Attachment.FILE_EXTENSION) Optional<String> fileExtension,
+                          @JsonProperty(JsonMessageConstants.Attachment.CONTENT_DISPOSITION) Optional<String> contentDisposition) {
+    public static MimePartDto from(MimePart mimePart) {
+        return new MimePartDto(mimePart.getTextualBody(),
+            mimePart.getMediaType(),
+            mimePart.getSubType(),
+            mimePart.getFileName(),
+            mimePart.getFileExtension(),
+            mimePart.getContentDisposition());
     }
 }
+
diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/SerializableMessage.java b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/SubjectsDto.java
similarity index 90%
copy from mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/SerializableMessage.java
copy to mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/SubjectsDto.java
index e91a295066..365e1e0056 100644
--- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/SerializableMessage.java
+++ b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/SubjectsDto.java
@@ -19,7 +19,9 @@
 
 package org.apache.james.mailbox.opensearch.json;
 
-public interface SerializableMessage {
+import java.util.Set;
 
-    String serialize();
+import com.fasterxml.jackson.annotation.JsonValue;
+
+public record SubjectsDto(@JsonValue Set<String> subjects) {
 }
diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/CriterionConverter.java b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/CriterionConverter.java
index 87063c9bf2..714d53bee3 100644
--- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/CriterionConverter.java
+++ b/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/query/CriterionConverter.java
@@ -38,8 +38,8 @@ import org.apache.james.mailbox.model.SearchQuery;
 import org.apache.james.mailbox.model.SearchQuery.Criterion;
 import org.apache.james.mailbox.model.SearchQuery.HeaderOperator;
 import org.apache.james.mailbox.opensearch.OpenSearchMailboxConfiguration;
-import org.apache.james.mailbox.opensearch.json.HeaderCollection;
 import org.apache.james.mailbox.opensearch.json.JsonMessageConstants;
+import org.apache.james.mailbox.store.search.mime.HeaderCollection;
 import org.opensearch.client.json.JsonData;
 import org.opensearch.client.opensearch._types.FieldValue;
 import org.opensearch.client.opensearch._types.query_dsl.BoolQuery;
diff --git a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/IndexableMessageTest.java b/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/IndexableMessageTest.java
index 19a32ab087..04385ae896 100644
--- a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/IndexableMessageTest.java
+++ b/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/IndexableMessageTest.java
@@ -289,7 +289,7 @@ class IndexableMessageTest {
         // Then
         String noTextualBody = "The textual body is not present";
         assertThat(indexableMessage.getAttachments())
-            .extracting(input -> input.getTextualBody().orElse(noTextualBody))
+            .extracting(input -> input.bodyTextContent().orElse(noTextualBody))
             .contains("first attachment content", noTextualBody, "third attachment content");
     }
 
diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/EMailer.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/EMailer.java
similarity index 91%
rename from mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/EMailer.java
rename to mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/EMailer.java
index c035e8583f..19d68e136e 100644
--- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/EMailer.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/EMailer.java
@@ -17,12 +17,11 @@
  * under the License.                                           *
  ****************************************************************/
 
-package org.apache.james.mailbox.opensearch.json;
+package org.apache.james.mailbox.store.search.mime;
 
 import java.util.Objects;
 import java.util.Optional;
 
-import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Joiner;
 import com.google.common.base.MoreObjects;
 
@@ -48,17 +47,14 @@ public class EMailer implements SerializableMessage {
         return s;
     }
 
-    @JsonProperty(JsonMessageConstants.EMailer.NAME)
     public Optional<String> getName() {
         return name;
     }
 
-    @JsonProperty(JsonMessageConstants.EMailer.ADDRESS)
     public String getAddress() {
         return address;
     }
 
-    @JsonProperty(JsonMessageConstants.EMailer.DOMAIN)
     public String getDomain() {
         return domain;
     }
diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/EMailers.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/EMailers.java
similarity index 94%
rename from mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/EMailers.java
rename to mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/EMailers.java
index d1a572c86b..fce4c191af 100644
--- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/EMailers.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/EMailers.java
@@ -17,12 +17,11 @@
  * under the License.                                           *
  ****************************************************************/
 
-package org.apache.james.mailbox.opensearch.json;
+package org.apache.james.mailbox.store.search.mime;
 
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.fasterxml.jackson.annotation.JsonValue;
 import com.google.common.base.Preconditions;
 
 public class EMailers implements SerializableMessage {
@@ -38,7 +37,6 @@ public class EMailers implements SerializableMessage {
         this.emailers = emailers;
     }
 
-    @JsonValue
     public Set<EMailer> getEmailers() {
         return emailers;
     }
diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/HeaderCollection.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/HeaderCollection.java
similarity index 97%
rename from mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/HeaderCollection.java
rename to mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/HeaderCollection.java
index ce3d0dfb7e..6066cf6e1f 100644
--- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/HeaderCollection.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/HeaderCollection.java
@@ -17,7 +17,7 @@
  * under the License.                                           *
  ****************************************************************/
 
-package org.apache.james.mailbox.opensearch.json;
+package org.apache.james.mailbox.store.search.mime;
 
 import java.time.ZonedDateTime;
 import java.util.List;
@@ -32,7 +32,6 @@ import org.apache.james.mime4j.field.address.LenientAddressParser;
 import org.apache.james.mime4j.stream.Field;
 import org.apache.james.mime4j.util.MimeUtil;
 
-import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
@@ -48,12 +47,10 @@ public class HeaderCollection {
             this.value = value;
         }
 
-        @JsonProperty(JsonMessageConstants.HEADER.NAME)
         public String getHeaderName() {
             return headerName;
         }
 
-        @JsonProperty(JsonMessageConstants.HEADER.VALUE)
         public String getValue() {
             return value;
         }
diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MimePart.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/MimePart.java
similarity index 95%
rename from mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MimePart.java
rename to mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/MimePart.java
index b400929f71..6e4732cc6e 100644
--- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MimePart.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/MimePart.java
@@ -17,7 +17,7 @@
  * under the License.                                           *
  ****************************************************************/
 
-package org.apache.james.mailbox.opensearch.json;
+package org.apache.james.mailbox.store.search.mime;
 
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
@@ -39,8 +39,6 @@ import org.apache.james.mime4j.stream.Field;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
 import com.github.fge.lambdas.Throwing;
 import com.google.common.collect.Lists;
 
@@ -249,53 +247,43 @@ public class MimePart {
         this.bodyTextContent = bodyTextContent;
     }
 
-    @JsonIgnore
     public List<MimePart> getAttachments() {
         return attachments;
     }
 
-    @JsonIgnore
     public HeaderCollection getHeaderCollection() {
         return headerCollection;
     }
 
-    @JsonProperty(JsonMessageConstants.Attachment.FILENAME)
     public Optional<String> getFileName() {
         return fileName;
     }
 
-    @JsonProperty(JsonMessageConstants.Attachment.FILE_EXTENSION)
     public Optional<String> getFileExtension() {
         return fileExtension;
     }
 
-    @JsonProperty(JsonMessageConstants.Attachment.MEDIA_TYPE)
     public Optional<String> getMediaType() {
         return mediaType.map(MediaType::asString);
     }
 
-    @JsonProperty(JsonMessageConstants.Attachment.SUBTYPE)
     public Optional<String> getSubType() {
         return subType.map(SubType::asString);
     }
 
-    @JsonProperty(JsonMessageConstants.Attachment.CONTENT_DISPOSITION)
     public Optional<String> getContentDisposition() {
         return contentDisposition;
     }
 
-    @JsonProperty(JsonMessageConstants.Attachment.TEXT_CONTENT)
     public Optional<String> getTextualBody() {
         return bodyTextContent;
     }
 
-    @JsonIgnore
     public Optional<String> locateFirstTextBody() {
         return firstBody(textAttachments()
                 .filter(this::isPlainSubType));
     }
 
-    @JsonIgnore
     public Optional<String> locateFirstHtmlBody() {
         if (locateFirstTextBody().isEmpty()) {
             return firstBody(textAttachments()
@@ -339,7 +327,6 @@ public class MimePart {
                 .isPresent();
     }
 
-    @JsonIgnore
     public Stream<MimePart> getAttachmentsStream() {
         return attachments.stream()
                 .flatMap(mimePart -> Stream.concat(Stream.of(mimePart), mimePart.getAttachmentsStream()))
diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MimePartContainerBuilder.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/MimePartContainerBuilder.java
similarity index 97%
rename from mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MimePartContainerBuilder.java
rename to mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/MimePartContainerBuilder.java
index be742dea29..f3dc788f24 100644
--- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MimePartContainerBuilder.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/MimePartContainerBuilder.java
@@ -17,7 +17,7 @@
  * under the License.                                           *
  ****************************************************************/
 
-package org.apache.james.mailbox.opensearch.json;
+package org.apache.james.mailbox.store.search.mime;
 
 import java.io.InputStream;
 import java.nio.charset.Charset;
diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MimePartParser.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/MimePartParser.java
similarity index 99%
rename from mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MimePartParser.java
rename to mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/MimePartParser.java
index 93e6ad7d99..827a2270c3 100644
--- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/MimePartParser.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/MimePartParser.java
@@ -17,7 +17,7 @@
  * under the License.                                           *
  ****************************************************************/
 
-package org.apache.james.mailbox.opensearch.json;
+package org.apache.james.mailbox.store.search.mime;
 
 import java.io.IOException;
 import java.io.InputStream;
diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/RootMimePartContainerBuilder.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/RootMimePartContainerBuilder.java
similarity index 98%
rename from mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/RootMimePartContainerBuilder.java
rename to mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/RootMimePartContainerBuilder.java
index 90e686b590..e087c59fbb 100644
--- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/RootMimePartContainerBuilder.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/RootMimePartContainerBuilder.java
@@ -17,7 +17,7 @@
  * under the License.                                           *
  ****************************************************************/
 
-package org.apache.james.mailbox.opensearch.json;
+package org.apache.james.mailbox.store.search.mime;
 
 import java.io.InputStream;
 import java.nio.charset.Charset;
diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/SerializableMessage.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/SerializableMessage.java
similarity index 96%
rename from mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/SerializableMessage.java
rename to mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/SerializableMessage.java
index e91a295066..3577e98262 100644
--- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/SerializableMessage.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/SerializableMessage.java
@@ -17,7 +17,7 @@
  * under the License.                                           *
  ****************************************************************/
 
-package org.apache.james.mailbox.opensearch.json;
+package org.apache.james.mailbox.store.search.mime;
 
 public interface SerializableMessage {
 
diff --git a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/Subjects.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/Subjects.java
similarity index 94%
rename from mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/Subjects.java
rename to mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/Subjects.java
index 3a8600521b..5d500f5ac9 100644
--- a/mailbox/opensearch/src/main/java/org/apache/james/mailbox/opensearch/json/Subjects.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/mime/Subjects.java
@@ -17,11 +17,10 @@
  * under the License.                                           *
  ****************************************************************/
 
-package org.apache.james.mailbox.opensearch.json;
+package org.apache.james.mailbox.store.search.mime;
 
 import java.util.Set;
 
-import com.fasterxml.jackson.annotation.JsonValue;
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 
@@ -38,7 +37,6 @@ public class Subjects implements SerializableMessage {
         this.subjects = subjects;
     }
 
-    @JsonValue
     public Set<String> getSubjects() {
         return subjects;
     }
diff --git a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/EMailerTest.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/mime/EMailerTest.java
similarity index 97%
rename from mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/EMailerTest.java
rename to mailbox/store/src/test/java/org/apache/james/mailbox/store/search/mime/EMailerTest.java
index f3790ae555..36e1e26b09 100644
--- a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/EMailerTest.java
+++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/mime/EMailerTest.java
@@ -17,7 +17,7 @@
  * under the License.                                           *
  ****************************************************************/
 
-package org.apache.james.mailbox.opensearch.json;
+package org.apache.james.mailbox.store.search.mime;
 
 import static org.assertj.core.api.Assertions.assertThatCode;
 
diff --git a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/EMailersTest.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/mime/EMailersTest.java
similarity index 98%
rename from mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/EMailersTest.java
rename to mailbox/store/src/test/java/org/apache/james/mailbox/store/search/mime/EMailersTest.java
index 33e22fd977..19e4912470 100644
--- a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/EMailersTest.java
+++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/mime/EMailersTest.java
@@ -17,7 +17,7 @@
  * under the License.                                           *
  ****************************************************************/
 
-package org.apache.james.mailbox.opensearch.json;
+package org.apache.james.mailbox.store.search.mime;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
diff --git a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/FieldImpl.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/mime/FieldImpl.java
similarity index 97%
rename from mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/FieldImpl.java
rename to mailbox/store/src/test/java/org/apache/james/mailbox/store/search/mime/FieldImpl.java
index 6a7b25340b..cf7c0450df 100644
--- a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/FieldImpl.java
+++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/mime/FieldImpl.java
@@ -17,7 +17,7 @@
  * under the License.                                           *
  ****************************************************************/
 
-package org.apache.james.mailbox.opensearch.json;
+package org.apache.james.mailbox.store.search.mime;
 
 import java.util.Objects;
 
diff --git a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/HeaderCollectionTest.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/mime/HeaderCollectionTest.java
similarity index 99%
rename from mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/HeaderCollectionTest.java
rename to mailbox/store/src/test/java/org/apache/james/mailbox/store/search/mime/HeaderCollectionTest.java
index 6d235d155d..1c75f898f1 100644
--- a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/HeaderCollectionTest.java
+++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/mime/HeaderCollectionTest.java
@@ -17,7 +17,7 @@
  * under the License.                                           *
  ****************************************************************/
 
-package org.apache.james.mailbox.opensearch.json;
+package org.apache.james.mailbox.store.search.mime;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
diff --git a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/MimePartTest.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/mime/MimePartTest.java
similarity index 97%
rename from mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/MimePartTest.java
rename to mailbox/store/src/test/java/org/apache/james/mailbox/store/search/mime/MimePartTest.java
index 38e7246e2a..cf80c8feff 100644
--- a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/MimePartTest.java
+++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/mime/MimePartTest.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations      *
  * under the License.                                           *
  ****************************************************************/
-package org.apache.james.mailbox.opensearch.json;
+package org.apache.james.mailbox.store.search.mime;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
diff --git a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/SubjectsTest.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/mime/SubjectsTest.java
similarity index 97%
rename from mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/SubjectsTest.java
rename to mailbox/store/src/test/java/org/apache/james/mailbox/store/search/mime/SubjectsTest.java
index 7779df597b..8c7bb08fc9 100644
--- a/mailbox/opensearch/src/test/java/org/apache/james/mailbox/opensearch/json/SubjectsTest.java
+++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/mime/SubjectsTest.java
@@ -17,7 +17,7 @@
  * under the License.                                           *
  ****************************************************************/
 
-package org.apache.james.mailbox.opensearch.json;
+package org.apache.james.mailbox.store.search.mime;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;


---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org


(james-project) 02/07: JAMES-4082 Introduce LuceneIndexableDocument

Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git

commit a4191571ef6ae83eee08b3d51778db80f7c3984d
Author: TungTV <vt...@linagora.com>
AuthorDate: Mon Nov 4 08:29:24 2024 +0700

    JAMES-4082 Introduce LuceneIndexableDocument
---
 .../lucene/search/DocumentFieldConstants.java      | 253 ++++++++++++++++++
 .../lucene/search/LuceneIndexableDocument.java     | 289 +++++++++++++++++++++
 2 files changed, 542 insertions(+)

diff --git a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/DocumentFieldConstants.java b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/DocumentFieldConstants.java
new file mode 100644
index 0000000000..f9bce07851
--- /dev/null
+++ b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/DocumentFieldConstants.java
@@ -0,0 +1,253 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one   *
+ * or more contributor license agreements.  See the NOTICE file *
+ * distributed with this work for additional information        *
+ * regarding copyright ownership.  The ASF licenses this file   *
+ * to you under the Apache License, Version 2.0 (the            *
+ * "License"); you may not use this file except in compliance   *
+ * with the License.  You may obtain a copy of the License at   *
+ *                                                              *
+ *   http://www.apache.org/licenses/LICENSE-2.0                 *
+ *                                                              *
+ * Unless required by applicable law or agreed to in writing,   *
+ * software distributed under the License is distributed on an  *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
+ * KIND, either express or implied.  See the License for the    *
+ * specific language governing permissions and limitations      *
+ * under the License.                                           *
+ ****************************************************************/
+
+package org.apache.james.mailbox.lucene.search;
+
+import java.util.function.Function;
+
+import jakarta.mail.Flags;
+
+import org.apache.james.mailbox.MailboxSession;
+import org.apache.james.mailbox.model.Mailbox;
+import org.apache.james.mailbox.model.MessageId;
+import org.apache.james.mailbox.store.mail.model.MailboxMessage;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+
+public interface DocumentFieldConstants {
+
+    /**
+     * {@link Field} which will contain the unique index of the {@link Document}
+     */
+    String ID_FIELD = "id";
+
+
+    /**
+     * {@link Field} which will contain uid of the {@link MailboxMessage}
+     */
+    String UID_FIELD = "uid";
+
+    /**
+     * {@link Field} boolean field that say if the message as an attachment or not
+     */
+    String HAS_ATTACHMENT_FIELD = "hasAttachment";
+
+    /**
+     * {@link Field} which will contain the {@link Flags} of the {@link MailboxMessage}
+     */
+    String FLAGS_FIELD = "flags";
+
+    /**
+     * {@link Field} which will contain the size of the {@link MailboxMessage}
+     */
+    String SIZE_FIELD = "size";
+
+    /**
+     * {@link Field} which will contain the body of the {@link MailboxMessage}
+     */
+    String BODY_FIELD = "body";
+
+    /**
+     * Prefix which will be used for each message header to store it also in a seperate {@link Field}
+     */
+    String PREFIX_HEADER_FIELD = "header_";
+
+    /**
+     * {@link Field} which will contain the whole message header of the {@link MailboxMessage}
+     */
+    String HEADERS_FIELD = "headers";
+
+    /**
+     * {@link Field} which will contain the mod-sequence of the message
+     */
+    String MODSEQ_FIELD = "modSeq";
+
+    /**
+     * {@link Field} which will contain the threadId of the message
+     */
+    String THREAD_ID_FIELD = "threadId";
+
+    /**
+     * {@link Field} which will contain the TO-Address of the message
+     */
+    String TO_FIELD = "to";
+
+    String FIRST_TO_MAILBOX_NAME_FIELD = "firstToMailboxName";
+
+    /**
+     * {@link Field} which will contain the CC-Address of the message
+     */
+    String CC_FIELD = "cc";
+
+    String FIRST_CC_MAILBOX_NAME_FIELD = "firstCcMailboxName";
+
+
+    /**
+     * {@link Field} which will contain the FROM-Address of the message
+     */
+    String FROM_FIELD = "from";
+
+    String FIRST_FROM_MAILBOX_NAME_FIELD = "firstFromMailboxName";
+
+    /**
+     * {@link Field} which will contain the BCC-Address of the message
+     */
+    String BCC_FIELD = "bcc";
+
+
+    String BASE_SUBJECT_FIELD = "baseSubject";
+    String SUBJECT_FIELD = "subject";
+
+    /**
+     * {@link Field} which contain the internalDate of the message with YEAR-Resolution
+     */
+    String INTERNAL_DATE_FIELD_YEAR_RESOLUTION = "internaldateYearResolution";
+
+
+    /**
+     * {@link Field} which contain the internalDate of the message with MONTH-Resolution
+     */
+    String INTERNAL_DATE_FIELD_MONTH_RESOLUTION = "internaldateMonthResolution";
+
+    /**
+     * {@link Field} which contain the internalDate of the message with DAY-Resolution
+     */
+    String INTERNAL_DATE_FIELD_DAY_RESOLUTION = "internaldateDayResolution";
+
+    /**
+     * {@link Field} which contain the internalDate of the message with HOUR-Resolution
+     */
+    String INTERNAL_DATE_FIELD_HOUR_RESOLUTION = "internaldateHourResolution";
+
+    /**
+     * {@link Field} which contain the internalDate of the message with MINUTE-Resolution
+     */
+    String INTERNAL_DATE_FIELD_MINUTE_RESOLUTION = "internaldateMinuteResolution";
+
+    /**
+     * {@link Field} which contain the internalDate of the message with SECOND-Resolution
+     */
+    String INTERNAL_DATE_FIELD_SECOND_RESOLUTION = "internaldateSecondResolution";
+
+
+    /**
+     * {@link Field} which contain the internalDate of the message with MILLISECOND-Resolution
+     */
+    String INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION = "internaldateMillisecondResolution";
+
+    /**
+     * {@link Field} which contain the saveDate of the message with YEAR-Resolution
+     */
+    String SAVE_DATE_FIELD_YEAR_RESOLUTION = "saveDateYearResolution";
+
+    /**
+     * {@link Field} which contain the saveDate of the message with MONTH-Resolution
+     */
+    String SAVE_DATE_FIELD_MONTH_RESOLUTION = "saveDateMonthResolution";
+
+    /**
+     * {@link Field} which contain the saveDate of the message with DAY-Resolution
+     */
+    String SAVE_DATE_FIELD_DAY_RESOLUTION = "saveDateDayResolution";
+
+    /**
+     * {@link Field} which contain the saveDate of the message with HOUR-Resolution
+     */
+    String SAVE_DATE_FIELD_HOUR_RESOLUTION = "saveDateHourResolution";
+
+    /**
+     * {@link Field} which contain the saveDate of the message with MINUTE-Resolution
+     */
+    String SAVE_DATE_FIELD_MINUTE_RESOLUTION = "saveDateMinuteResolution";
+
+    /**
+     * {@link Field} which contain the saveDate of the message with SECOND-Resolution
+     */
+    String SAVE_DATE_FIELD_SECOND_RESOLUTION = "saveDateSecondResolution";
+
+    /**
+     * {@link Field} which will contain the id of the {@link Mailbox}
+     */
+    String MAILBOX_ID_FIELD = "mailboxid";
+
+    /**
+     * {@link Field} which will contain the user of the {@link MailboxSession}
+     */
+    String USERS = "userSession";
+    /**
+     * {@link Field} which will contain the id of the {@link MessageId}
+     */
+    String MESSAGE_ID_FIELD = "messageid";
+
+    /**
+     * {@link Field} which contain the Date header of the message with YEAR-Resolution
+     */
+    String SENT_DATE_FIELD_YEAR_RESOLUTION = "sentdateYearResolution";
+
+
+    /**
+     * {@link Field} which contain the Date header of the message with MONTH-Resolution
+     */
+    String SENT_DATE_FIELD_MONTH_RESOLUTION = "sentdateMonthResolution";
+
+    /**
+     * {@link Field} which contain the Date header of the message with DAY-Resolution
+     */
+    String SENT_DATE_FIELD_DAY_RESOLUTION = "sentdateDayResolution";
+
+    /**
+     * {@link Field} which contain the Date header of the message with HOUR-Resolution
+     */
+    String SENT_DATE_FIELD_HOUR_RESOLUTION = "sentdateHourResolution";
+
+    /**
+     * {@link Field} which contain the Date header of the message with MINUTE-Resolution
+     */
+    String SENT_DATE_FIELD_MINUTE_RESOLUTION = "sentdateMinuteResolution";
+
+    /**
+     * {@link Field} which contain the Date header of the message with SECOND-Resolution
+     */
+    String SENT_DATE_FIELD_SECOND_RESOLUTION = "sentdateSecondResolution";
+
+
+    /**
+     * {@link Field} which contain the Date header of the message with MILLISECOND-Resolution
+     */
+    String SENT_DATE_FIELD_MILLISECOND_RESOLUTION = "sentdateMillisecondResolution";
+
+    String SENT_DATE_SORT_FIELD_MILLISECOND_RESOLUTION = "sentdateSort";
+
+    String ATTACHMENTS = "attachments";
+
+    interface Attachment {
+        String TEXT_CONTENT = "textContent";
+        String MEDIA_TYPE = "mediaType";
+        String SUBTYPE = "subtype";
+        String CONTENT_DISPOSITION = "contentDisposition";
+        String FILENAME = "fileName";
+        String FILE_EXTENSION = "fileExtension";
+    }
+
+    Function<String, String> ATTACHMENT_FIELD_FUNCTION = (fieldName) -> ATTACHMENTS + "." + fieldName;
+
+    String ATTACHMENT_TEXT_CONTENT_FIELD = ATTACHMENT_FIELD_FUNCTION.apply(Attachment.TEXT_CONTENT);
+    String ATTACHMENT_FILE_NAME_FIELD = ATTACHMENT_FIELD_FUNCTION.apply(Attachment.FILENAME);
+
+}
diff --git a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneIndexableDocument.java b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneIndexableDocument.java
new file mode 100644
index 0000000000..1cb006783b
--- /dev/null
+++ b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneIndexableDocument.java
@@ -0,0 +1,289 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one   *
+ * or more contributor license agreements.  See the NOTICE file *
+ * distributed with this work for additional information        *
+ * regarding copyright ownership.  The ASF licenses this file   *
+ * to you under the Apache License, Version 2.0 (the            *
+ * "License"); you may not use this file except in compliance   *
+ * with the License.  You may obtain a copy of the License at   *
+ *                                                              *
+ *   http://www.apache.org/licenses/LICENSE-2.0                 *
+ *                                                              *
+ * Unless required by applicable law or agreed to in writing,   *
+ * software distributed under the License is distributed on an  *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
+ * KIND, either express or implied.  See the License for the    *
+ * specific language governing permissions and limitations      *
+ * under the License.                                           *
+ ****************************************************************/
+
+package org.apache.james.mailbox.lucene.search;
+
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.ATTACHMENT_FILE_NAME_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.ATTACHMENT_TEXT_CONTENT_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.BASE_SUBJECT_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.BCC_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.BODY_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.CC_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.FIRST_CC_MAILBOX_NAME_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.FIRST_FROM_MAILBOX_NAME_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.FIRST_TO_MAILBOX_NAME_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.FLAGS_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.FROM_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.HAS_ATTACHMENT_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.HEADERS_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.ID_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.INTERNAL_DATE_FIELD_DAY_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.INTERNAL_DATE_FIELD_HOUR_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.INTERNAL_DATE_FIELD_MINUTE_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.INTERNAL_DATE_FIELD_MONTH_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.INTERNAL_DATE_FIELD_SECOND_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.INTERNAL_DATE_FIELD_YEAR_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.MAILBOX_ID_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.MESSAGE_ID_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.PREFIX_HEADER_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SAVE_DATE_FIELD_DAY_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SAVE_DATE_FIELD_HOUR_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SAVE_DATE_FIELD_MINUTE_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SAVE_DATE_FIELD_MONTH_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SAVE_DATE_FIELD_SECOND_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SAVE_DATE_FIELD_YEAR_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SENT_DATE_FIELD_DAY_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SENT_DATE_FIELD_HOUR_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SENT_DATE_FIELD_MILLISECOND_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SENT_DATE_FIELD_MINUTE_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SENT_DATE_FIELD_MONTH_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SENT_DATE_FIELD_SECOND_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SENT_DATE_FIELD_YEAR_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SENT_DATE_SORT_FIELD_MILLISECOND_RESOLUTION;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SIZE_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.SUBJECT_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.THREAD_ID_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.TO_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.UID_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.USERS;
+
+import java.util.Arrays;
+import java.util.Date;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+
+import jakarta.mail.Flags;
+
+import org.apache.james.mailbox.MailboxSession;
+import org.apache.james.mailbox.MessageUid;
+import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.model.MailboxId;
+import org.apache.james.mailbox.model.MessageAttachmentMetadata;
+import org.apache.james.mailbox.store.mail.model.MailboxMessage;
+import org.apache.james.mailbox.store.search.SearchUtil;
+import org.apache.james.mailbox.store.search.mime.EMailers;
+import org.apache.james.mailbox.store.search.mime.HeaderCollection;
+import org.apache.james.mailbox.store.search.mime.MimePart;
+import org.apache.james.mailbox.store.search.mime.MimePartParser;
+import org.apache.lucene.document.DateTools;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.util.BytesRef;
+
+import com.github.fge.lambdas.Throwing;
+import com.google.common.collect.ImmutableMap;
+
+import reactor.core.publisher.Mono;
+
+public class LuceneIndexableDocument {
+    public static final Map<Flags.Flag, String> SYSTEM_FLAG_STRING_MAP = ImmutableMap.<Flags.Flag, String>builder()
+        .put(Flags.Flag.ANSWERED, "\\ANSWERED")
+        .put(Flags.Flag.DELETED, "\\DELETED")
+        .put(Flags.Flag.DRAFT, "\\DRAFT")
+        .put(Flags.Flag.FLAGGED, "\\FLAGGED")
+        .put(Flags.Flag.RECENT, "\\RECENT")
+        .put(Flags.Flag.SEEN, "\\FLAG")
+        .build();
+
+    private final TextExtractor textExtractor;
+
+    public LuceneIndexableDocument(TextExtractor textExtractor) {
+        this.textExtractor = textExtractor;
+    }
+
+    public Mono<Document> createMessageDocument(MailboxMessage message, MailboxSession session) {
+        return Throwing.supplier(() -> new MimePartParser(textExtractor).parse(message.getFullContent())).get()
+            .asMimePart(textExtractor)
+            .map(parsingResult -> createMessageDocument(message, session, parsingResult));
+    }
+
+    /**
+     * Create a new {@link Document} for the given {@link MailboxMessage}. This Document does not contain any flags data. The {@link Flags} are stored in a seperate Document.
+     * <p>
+     * See {@link #createFlagsDocument(MailboxMessage)}
+     */
+    public Document createMessageDocument(MailboxMessage message, MailboxSession session, MimePart mimePartExtracted) {
+        Document doc = new Document();
+        doc.add(new StringField(USERS, session.getUser().asString().toUpperCase(Locale.US), Field.Store.YES));
+        doc.add(new StringField(MAILBOX_ID_FIELD, message.getMailboxId().serialize().toUpperCase(Locale.US), Field.Store.YES));
+        doc.add(new NumericDocValuesField(UID_FIELD, message.getUid().asLong()));
+        doc.add(new LongPoint(UID_FIELD, message.getUid().asLong()));
+        doc.add(new StoredField(UID_FIELD, message.getUid().asLong()));
+        doc.add(new StringField(HAS_ATTACHMENT_FIELD, Boolean.toString(MessageAttachmentMetadata.hasNonInlinedAttachment(message.getAttachments())), Field.Store.YES));
+        doc.add(new LongPoint(SIZE_FIELD, message.getFullContentOctets()));
+        doc.add(new NumericDocValuesField(SIZE_FIELD, message.getFullContentOctets()));
+
+        // create a unique key for the document which can be used later on updates to find the document
+        doc.add(new StringField(ID_FIELD, message.getMailboxId().serialize().toUpperCase(Locale.US) + "-" + message.getUid().asLong(), Field.Store.YES));
+
+        Optional.ofNullable(SearchUtil.getSerializedMessageIdIfSupportedByUnderlyingStorageOrNull(message))
+            .ifPresent(serializedMessageId -> doc.add(new StringField(MESSAGE_ID_FIELD, serializedMessageId, Field.Store.YES)));
+        Optional.ofNullable(SearchUtil.getSerializedThreadIdIfSupportedByUnderlyingStorageOrNull(message))
+            .ifPresent(serializedThreadId -> doc.add(new StringField(THREAD_ID_FIELD, serializedThreadId, Field.Store.YES)));
+
+        HeaderCollection headerCollection = mimePartExtracted.getHeaderCollection();
+
+        // index date fields
+        indexInternalDateFields(message.getInternalDate(), doc);
+        message.getSaveDate().ifPresent(saveDate -> indexSaveDateFields(saveDate, doc));
+        headerCollection.getSentDate()
+            .map(zonedDateTime -> Date.from(zonedDateTime.toInstant()))
+            .or(() -> Optional.ofNullable(message.getInternalDate()))
+            .ifPresent(sentDate -> indexSentDateFields(sentDate, doc));
+
+        // index header
+        headerCollection.getHeaders()
+            .forEach(header -> {
+                String headerName = uppercase(header.getHeaderName());
+                String headerValue = uppercase(header.getValue());
+                doc.add(new TextField(HEADERS_FIELD, String.format("%s: %s", headerName, headerValue), Field.Store.NO));
+                doc.add(new TextField(PREFIX_HEADER_FIELD + headerName, headerValue, Field.Store.NO));
+
+                switch (headerName) {
+                    case "TO":
+                        doc.add(new TextField(TO_FIELD, headerValue, Field.Store.NO));
+                        doc.add(new SortedSetDocValuesField(FIRST_TO_MAILBOX_NAME_FIELD, new BytesRef(SearchUtil.getMailboxAddress(header.getValue()))));
+                        break;
+                    case "FROM":
+                        doc.add(new TextField(FROM_FIELD, headerValue, Field.Store.NO));
+                        doc.add(new SortedSetDocValuesField(FIRST_FROM_MAILBOX_NAME_FIELD, new BytesRef(SearchUtil.getMailboxAddress(header.getValue()))));
+                        break;
+                    case "CC":
+                        doc.add(new TextField(CC_FIELD, headerValue, Field.Store.NO));
+                        doc.add(new SortedSetDocValuesField(FIRST_CC_MAILBOX_NAME_FIELD, new BytesRef(SearchUtil.getMailboxAddress(header.getValue()))));
+                        break;
+                    case "BCC":
+                        doc.add(new TextField(BCC_FIELD, headerValue, Field.Store.NO));
+                        break;
+                    case "SUBJECT":
+                        doc.add(new StringField(SUBJECT_FIELD, header.getValue(), Field.Store.YES));
+                        doc.add(new StringField(BASE_SUBJECT_FIELD, headerValue, Field.Store.NO));
+                        doc.add(new SortedSetDocValuesField(BASE_SUBJECT_FIELD, new BytesRef(SearchUtil.getBaseSubject(headerValue))));
+                        break;
+                    default:
+                        break;
+                }
+            });
+
+        doc.add(new TextField(FROM_FIELD, uppercase(EMailers.from(headerCollection.getFromAddressSet()).serialize()), Field.Store.YES));
+        doc.add(new TextField(TO_FIELD, uppercase(EMailers.from(headerCollection.getToAddressSet()).serialize()), Field.Store.YES));
+        doc.add(new TextField(CC_FIELD, uppercase(EMailers.from(headerCollection.getCcAddressSet()).serialize()), Field.Store.YES));
+        doc.add(new TextField(BCC_FIELD, uppercase(EMailers.from(headerCollection.getBccAddressSet()).serialize()), Field.Store.YES));
+
+        // index body
+        Optional<String> bodyText = mimePartExtracted.locateFirstTextBody();
+        Optional<String> bodyHtml = mimePartExtracted.locateFirstHtmlBody();
+
+        bodyText.or(() -> bodyHtml)
+            .ifPresent(bodyContent -> doc.add(new TextField(BODY_FIELD, bodyContent, Field.Store.YES)));
+
+        // index attachment
+        mimePartExtracted.getAttachmentsStream().forEach(attachmentFields -> {
+            attachmentFields.getTextualBody().ifPresent(textualBody -> doc.add(new TextField(ATTACHMENT_TEXT_CONTENT_FIELD, textualBody, Field.Store.YES)));
+            attachmentFields.getFileName().ifPresent(fileName -> doc.add(new StringField(ATTACHMENT_FILE_NAME_FIELD, uppercase(fileName), Field.Store.YES)));
+        });
+        return doc;
+    }
+
+    /**
+     * Index the {@link Flags} and add it to the {@link Document}
+     */
+    public Document createFlagsDocument(MailboxMessage message) {
+        return createFlagsDocument(message.getMailboxId(), message.getUid(), message.createFlags());
+    }
+
+    public Document createFlagsDocument(MailboxId mailboxId, final MessageUid messageUid, Flags messageFlags) {
+        Document doc = new Document();
+        doc.add(new StringField(ID_FIELD, createFlagsIdField(mailboxId, messageUid), Field.Store.YES));
+        doc.add(new StringField(MAILBOX_ID_FIELD, mailboxId.serialize(), Field.Store.YES));
+
+        Optional.of(messageUid.asLong())
+            .ifPresent(uidAsLong -> {
+                doc.add(new NumericDocValuesField(UID_FIELD, uidAsLong));
+                doc.add(new LongPoint(UID_FIELD, uidAsLong));
+                doc.add(new StoredField(UID_FIELD, uidAsLong));
+            });
+
+        Arrays.stream(messageFlags.getSystemFlags())
+            .forEach(sysFlag ->
+                doc.add(new StringField(FLAGS_FIELD, Optional.ofNullable(SYSTEM_FLAG_STRING_MAP.get(sysFlag))
+                    .orElse(sysFlag.toString()), Field.Store.YES)));
+
+        Arrays.stream(messageFlags.getUserFlags())
+            .forEach(userFlag -> doc.add(new StringField(FLAGS_FIELD, userFlag, Field.Store.YES)));
+
+        // if no flags are there we just use a empty field
+        if (messageFlags.getSystemFlags().length == 0 && messageFlags.getUserFlags().length == 0) {
+            doc.add(new StringField(FLAGS_FIELD, "", Field.Store.NO));
+        }
+
+        return doc;
+    }
+
+    public static String createFlagsIdField(MailboxId mailboxId, MessageUid messageUid) {
+        return "flags-" + mailboxId.serialize() + "-" + messageUid.asLong();
+    }
+
+    private static void indexSaveDateFields(Date saveDate, Document doc) {
+        doc.add(new StringField(SAVE_DATE_FIELD_YEAR_RESOLUTION, DateTools.dateToString(saveDate, DateTools.Resolution.YEAR), Field.Store.NO));
+        doc.add(new StringField(SAVE_DATE_FIELD_MONTH_RESOLUTION, DateTools.dateToString(saveDate, DateTools.Resolution.MONTH), Field.Store.NO));
+        doc.add(new StringField(SAVE_DATE_FIELD_DAY_RESOLUTION, DateTools.dateToString(saveDate, DateTools.Resolution.DAY), Field.Store.NO));
+        doc.add(new StringField(SAVE_DATE_FIELD_HOUR_RESOLUTION, DateTools.dateToString(saveDate, DateTools.Resolution.HOUR), Field.Store.NO));
+        doc.add(new StringField(SAVE_DATE_FIELD_MINUTE_RESOLUTION, DateTools.dateToString(saveDate, DateTools.Resolution.MINUTE), Field.Store.NO));
+        doc.add(new StringField(SAVE_DATE_FIELD_SECOND_RESOLUTION, DateTools.dateToString(saveDate, DateTools.Resolution.SECOND), Field.Store.NO));
+    }
+
+    private static void indexInternalDateFields(Date date, Document doc) {
+        doc.add(new StringField(INTERNAL_DATE_FIELD_YEAR_RESOLUTION, DateTools.dateToString(date, DateTools.Resolution.YEAR), Field.Store.NO));
+        doc.add(new StringField(INTERNAL_DATE_FIELD_MONTH_RESOLUTION, DateTools.dateToString(date, DateTools.Resolution.MONTH), Field.Store.NO));
+        doc.add(new StringField(INTERNAL_DATE_FIELD_DAY_RESOLUTION, DateTools.dateToString(date, DateTools.Resolution.DAY), Field.Store.NO));
+        doc.add(new StringField(INTERNAL_DATE_FIELD_HOUR_RESOLUTION, DateTools.dateToString(date, DateTools.Resolution.HOUR), Field.Store.NO));
+        doc.add(new StringField(INTERNAL_DATE_FIELD_MINUTE_RESOLUTION, DateTools.dateToString(date, DateTools.Resolution.MINUTE), Field.Store.NO));
+        doc.add(new StringField(INTERNAL_DATE_FIELD_SECOND_RESOLUTION, DateTools.dateToString(date, DateTools.Resolution.SECOND), Field.Store.NO));
+        doc.add(new NumericDocValuesField(INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION, Long.parseLong(DateTools.dateToString(date, DateTools.Resolution.MILLISECOND))));
+    }
+
+    private static void indexSentDateFields(Date sentDate, Document doc) {
+        doc.add(new StringField(SENT_DATE_FIELD_YEAR_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.YEAR), Field.Store.NO));
+        doc.add(new StringField(SENT_DATE_FIELD_MONTH_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.MONTH), Field.Store.NO));
+        doc.add(new StringField(SENT_DATE_FIELD_DAY_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.DAY), Field.Store.NO));
+        doc.add(new StringField(SENT_DATE_FIELD_HOUR_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.HOUR), Field.Store.NO));
+        doc.add(new StringField(SENT_DATE_FIELD_MINUTE_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.MINUTE), Field.Store.NO));
+        doc.add(new StringField(SENT_DATE_FIELD_SECOND_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.SECOND), Field.Store.NO));
+        doc.add(new StringField(SENT_DATE_FIELD_MILLISECOND_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.MILLISECOND), Field.Store.NO));
+        doc.add(new NumericDocValuesField(SENT_DATE_SORT_FIELD_MILLISECOND_RESOLUTION, Long.parseLong(DateTools.dateToString(sentDate, DateTools.Resolution.MILLISECOND))));
+    }
+
+    public static String uppercase(String value) {
+        return value.toUpperCase(Locale.US);
+    }
+
+    public static String lowercase(String value) {
+        return value.toLowerCase(Locale.US);
+    }
+
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org


(james-project) 06/07: JAMES-2182 Lucene search - remove SearchCapabilities.PartialEmailMatch (performance reason)

Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git

commit f4912da7b3a2dbbe8df548c1c94f43afd3016d4d
Author: TungTV <vt...@linagora.com>
AuthorDate: Mon Nov 4 15:01:43 2024 +0700

    JAMES-2182 Lucene search - remove SearchCapabilities.PartialEmailMatch (performance reason)
---
 .../apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
index 385db3d0a2..ab9de491f9 100644
--- a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
+++ b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
@@ -209,7 +209,7 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
 
     private int maxQueryResults = DEFAULT_MAX_QUERY_RESULTS;
 
-    private boolean suffixMatch = true;
+    private boolean suffixMatch = false;
 
     @Inject
     public LuceneMessageSearchIndex(
@@ -257,7 +257,6 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
     @Override
     public EnumSet<SearchCapabilities> getSupportedCapabilities(EnumSet<MailboxManager.MessageCapabilities> messageCapabilities) {
         return EnumSet.of(SearchCapabilities.MultimailboxSearch,
-            SearchCapabilities.PartialEmailMatch,
             SearchCapabilities.Text,
             SearchCapabilities.FullText,
             SearchCapabilities.AttachmentFileName,


---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org


(james-project) 03/07: JAMES-4082 Adapt Lucene search - support full capability

Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git

commit 956d5cc8b134785bb319417eb3796e5cdab8e408
Author: TungTV <vt...@linagora.com>
AuthorDate: Mon Nov 4 08:29:24 2024 +0700

    JAMES-4082 Adapt Lucene search - support full capability
---
 mailbox/lucene/pom.xml                             | 11 ++++
 .../lucene/search/LuceneIndexableDocument.java     |  2 +-
 .../lucene/search/LuceneMessageSearchIndex.java    | 67 +++++++++++++---------
 .../lucene/search/LuceneSearchHighlighter.java     | 18 +++++-
 .../search/LuceneFSSearchHighLighterTest.java      |  3 +-
 .../search/LuceneMailboxMessageFlagSearchTest.java |  3 +-
 .../LuceneMailboxMessageSearchIndexTest.java       |  3 +-
 .../search/LuceneMemorySearchHighLighterTest.java  | 58 ++++++++++++++++++-
 .../search/LuceneMessageSearchIndexTest.java       | 25 +++++++-
 .../lucenesearch/host/LuceneSearchHostSystem.java  |  3 +-
 10 files changed, 156 insertions(+), 37 deletions(-)

diff --git a/mailbox/lucene/pom.xml b/mailbox/lucene/pom.xml
index 484e20e302..7844c9709d 100644
--- a/mailbox/lucene/pom.xml
+++ b/mailbox/lucene/pom.xml
@@ -63,6 +63,17 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>${james.groupId}</groupId>
+            <artifactId>apache-james-mailbox-tika</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>${james.groupId}</groupId>
+            <artifactId>apache-james-mailbox-tika</artifactId>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>${james.groupId}</groupId>
             <artifactId>event-bus-api</artifactId>
diff --git a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneIndexableDocument.java b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneIndexableDocument.java
index 1cb006783b..bee20308d2 100644
--- a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneIndexableDocument.java
+++ b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneIndexableDocument.java
@@ -234,7 +234,7 @@ public class LuceneIndexableDocument {
                     .orElse(sysFlag.toString()), Field.Store.YES)));
 
         Arrays.stream(messageFlags.getUserFlags())
-            .forEach(userFlag -> doc.add(new StringField(FLAGS_FIELD, userFlag, Field.Store.YES)));
+            .forEach(userFlag -> doc.add(new StringField(FLAGS_FIELD, lowercase(userFlag), Field.Store.YES)));
 
         // if no flags are there we just use a empty field
         if (messageFlags.getSystemFlags().length == 0 && messageFlags.getUserFlags().length == 0) {
diff --git a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
index 12b47a3209..81afadae54 100644
--- a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
+++ b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
@@ -18,6 +18,9 @@
  ****************************************************************/
 package org.apache.james.mailbox.lucene.search;
 
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.ATTACHMENT_FILE_NAME_FIELD;
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.ATTACHMENT_TEXT_CONTENT_FIELD;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.Charset;
@@ -47,6 +50,7 @@ import org.apache.james.mailbox.MessageUid;
 import org.apache.james.mailbox.SessionProvider;
 import org.apache.james.mailbox.exception.MailboxException;
 import org.apache.james.mailbox.exception.UnsupportedSearchException;
+import org.apache.james.mailbox.extractor.TextExtractor;
 import org.apache.james.mailbox.model.Mailbox;
 import org.apache.james.mailbox.model.MailboxId;
 import org.apache.james.mailbox.model.MessageAttachmentMetadata;
@@ -394,6 +398,7 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
 
     private final MailboxId.Factory mailboxIdFactory;
     private final MessageId.Factory messageIdFactory;
+    private final LuceneIndexableDocument indexableDocument;
 
     @VisibleForTesting
     final IndexWriter writer;
@@ -401,7 +406,7 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
 
     private int maxQueryResults = DEFAULT_MAX_QUERY_RESULTS;
 
-    private boolean suffixMatch = false;
+    private boolean suffixMatch = true;
 
     @Inject
     public LuceneMessageSearchIndex(
@@ -409,8 +414,9 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
         MailboxId.Factory mailboxIdFactory,
         Directory directory,
         MessageId.Factory messageIdFactory,
-        SessionProvider sessionProvider) throws IOException {
-        this(factory, mailboxIdFactory, directory, false, messageIdFactory, sessionProvider);
+        SessionProvider sessionProvider,
+        TextExtractor textExtractor) throws IOException {
+        this(factory, mailboxIdFactory, directory, false, messageIdFactory, sessionProvider, textExtractor);
     }
 
     public LuceneMessageSearchIndex(
@@ -419,10 +425,12 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
             Directory directory,
             boolean dropIndexOnStart,
             MessageId.Factory messageIdFactory,
-            SessionProvider sessionProvider) throws IOException {
+            SessionProvider sessionProvider,
+            TextExtractor textExtractor) throws IOException {
         super(factory, ImmutableSet.of(), sessionProvider);
         this.mailboxIdFactory = mailboxIdFactory;
         this.messageIdFactory = messageIdFactory;
+        this.indexableDocument = new LuceneIndexableDocument(textExtractor);
         this.directory = directory;
         try {
             this.writer = new IndexWriter(this.directory, createConfig(LenientImapSearchAnalyzer.INSTANCE, dropIndexOnStart));
@@ -445,8 +453,13 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
 
     @Override
     public EnumSet<SearchCapabilities> getSupportedCapabilities(EnumSet<MailboxManager.MessageCapabilities> messageCapabilities) {
-        return EnumSet.of(SearchCapabilities.MultimailboxSearch);
-
+        return EnumSet.of(SearchCapabilities.MultimailboxSearch,
+            SearchCapabilities.PartialEmailMatch,
+            SearchCapabilities.Text,
+            SearchCapabilities.FullText,
+            SearchCapabilities.AttachmentFileName,
+            SearchCapabilities.Attachment,
+            SearchCapabilities.HighlightSearch);
     }
 
     /**
@@ -1041,19 +1054,20 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
     /**
      * Return a {@link Query} which is build based on the given {@link SearchQuery.TextCriterion}
      */
-    private Query createTextQuery(SearchQuery.TextCriterion crit) throws UnsupportedSearchException {
+    private Query createTextQuery(SearchQuery.TextCriterion crit) {
         String value = crit.getOperator().getValue().toUpperCase(Locale.US);
-        switch (crit.getType()) {
-        case BODY:
-            return createTermQuery(BODY_FIELD, value);
-        case FULL:
-            BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
-            queryBuilder.add(createTermQuery(BODY_FIELD, value), BooleanClause.Occur.SHOULD);
-            queryBuilder.add(createTermQuery(HEADERS_FIELD,value), BooleanClause.Occur.SHOULD);
-            return queryBuilder.build();
-        default:
-            throw new UnsupportedSearchException();
-        }
+        return switch (crit.getType()) {
+            case BODY -> createTermQuery(BODY_FIELD, value);
+            case ATTACHMENTS -> createTermQuery(ATTACHMENT_TEXT_CONTENT_FIELD, value);
+            case ATTACHMENT_FILE_NAME -> createTermQuery(ATTACHMENT_FILE_NAME_FIELD, value);
+            case FULL -> {
+                BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
+                queryBuilder.add(createTermQuery(BODY_FIELD, value), BooleanClause.Occur.SHOULD);
+                queryBuilder.add(createTermQuery(HEADERS_FIELD, value), BooleanClause.Occur.SHOULD);
+                queryBuilder.add(createTermQuery(ATTACHMENT_TEXT_CONTENT_FIELD, value), BooleanClause.Occur.SHOULD);
+                yield queryBuilder.build();
+            }
+        };
     }
 
     /**
@@ -1120,7 +1134,7 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
         } else if (criterion instanceof AttachmentCriterion crit) {
             return createAttachmentQuery(crit.getOperator().isSet());
         } else if (criterion instanceof CustomFlagCriterion crit) {
-            return createFlagQuery(crit.getFlag(), crit.getOperator().isSet(), inMailboxes, recentUids);
+            return createFlagQuery(crit.getFlag().toLowerCase(Locale.US), crit.getOperator().isSet(), inMailboxes, recentUids);
         } else if (criterion instanceof SearchQuery.TextCriterion crit) {
             return createTextQuery(crit);
         } else if (criterion instanceof SearchQuery.AllCriterion) {
@@ -1141,15 +1155,12 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
 
     @Override
     public Mono<Void> add(MailboxSession session, Mailbox mailbox, MailboxMessage membership) {
-        return Mono.fromRunnable(Throwing.runnable(() -> {
-            Document doc = createMessageDocument(session, membership);
-            Document flagsDoc = createFlagsDocument(membership);
-
-            log.trace("Adding document: uid:'{}' with flags: {}", doc.get("uid"), flagsDoc);
-
-            writer.addDocument(doc);
-            writer.addDocument(flagsDoc);
-        }));
+        return indexableDocument.createMessageDocument(membership, session)
+            .flatMap(document -> Mono.fromRunnable(Throwing.runnable(() -> {
+                writer.addDocument(document);
+                writer.addDocument(indexableDocument.createFlagsDocument(membership));
+            })))
+            .then();
     }
 
     @Override
diff --git a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneSearchHighlighter.java b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneSearchHighlighter.java
index ee1fdf6edc..7a6804988d 100644
--- a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneSearchHighlighter.java
+++ b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneSearchHighlighter.java
@@ -19,15 +19,18 @@
 
 package org.apache.james.mailbox.lucene.search;
 
+import static org.apache.james.mailbox.lucene.search.DocumentFieldConstants.ATTACHMENT_TEXT_CONTENT_FIELD;
 import static org.apache.james.mailbox.lucene.search.LuceneMessageSearchIndex.BODY_FIELD;
 import static org.apache.james.mailbox.lucene.search.LuceneMessageSearchIndex.MESSAGE_ID_FIELD;
 import static org.apache.james.mailbox.lucene.search.LuceneMessageSearchIndex.SUBJECT_FIELD;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Locale;
 import java.util.Optional;
+import java.util.stream.Stream;
 
 import org.apache.james.mailbox.MailboxSession;
 import org.apache.james.mailbox.model.MailboxId;
@@ -44,6 +47,7 @@ import org.apache.james.mailbox.store.StoreMailboxManager;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.queryparser.classic.QueryParser;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
@@ -62,6 +66,7 @@ import reactor.core.publisher.Mono;
 import reactor.core.scheduler.Schedulers;
 
 public class LuceneSearchHighlighter implements SearchHighlighter {
+
     private static Analyzer defaultAnalyzer() {
         return new StandardAnalyzer();
     }
@@ -156,8 +161,9 @@ public class LuceneSearchHighlighter implements SearchHighlighter {
 
     private SearchSnippet buildSearchSnippet(Document doc, SearchQuery searchQuery) throws IOException, InvalidTokenOffsetsException {
         MessageId messageId = messageIdFactory.fromString(doc.get(MESSAGE_ID_FIELD));
-        String highlightedSubject = getHighlightedSubject(doc, searchQuery);
-        String highlightedBody = getHighlightedBody(doc, searchQuery);
+        Optional<String> highlightedSubject = Optional.ofNullable(getHighlightedSubject(doc, searchQuery));
+        Optional<String> highlightedBody = Optional.ofNullable(getHighlightedBody(doc, searchQuery))
+            .or(() -> getHighlightAttachmentTextBody(doc, searchQuery));
 
         return new SearchSnippet(messageId, highlightedSubject, highlightedBody);
     }
@@ -169,4 +175,12 @@ public class LuceneSearchHighlighter implements SearchHighlighter {
     private String getHighlightedBody(Document doc, SearchQuery searchQuery) throws IOException, InvalidTokenOffsetsException {
         return highlighter(searchQuery).getBestFragment(analyzer, BODY_FIELD, doc.get(BODY_FIELD));
     }
+
+    private Optional<String> getHighlightAttachmentTextBody(Document doc, SearchQuery searchQuery) {
+        Highlighter highlighter = highlighter(searchQuery);
+        return Stream.ofNullable(doc.getFields(ATTACHMENT_TEXT_CONTENT_FIELD)).flatMap(Arrays::stream)
+            .map(IndexableField::stringValue)
+            .map(Throwing.function(contentType -> highlighter.getBestFragment(analyzer, ATTACHMENT_TEXT_CONTENT_FIELD, contentType)))
+            .findFirst();
+    }
 }
diff --git a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneFSSearchHighLighterTest.java b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneFSSearchHighLighterTest.java
index ea43b4b87a..850efeeb95 100644
--- a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneFSSearchHighLighterTest.java
+++ b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneFSSearchHighLighterTest.java
@@ -40,6 +40,7 @@ import org.apache.james.mailbox.searchhighligt.SearchHighlighter;
 import org.apache.james.mailbox.searchhighligt.SearchHighlighterConfiguration;
 import org.apache.james.mailbox.store.StoreMailboxManager;
 import org.apache.james.mailbox.store.StoreMessageManager;
+import org.apache.james.mailbox.store.extractor.JsoupTextExtractor;
 import org.apache.james.mailbox.store.search.MessageSearchIndex;
 import org.apache.lucene.store.FSDirectory;
 import org.junit.jupiter.api.BeforeEach;
@@ -65,7 +66,7 @@ class LuceneFSSearchHighLighterTest implements SearchHighLighterContract {
             .listeningSearchIndex(Throwing.function(preInstanciationStage -> new LuceneMessageSearchIndex(
                 preInstanciationStage.getMapperFactory(), new InMemoryId.Factory(), FSDirectory.open(tempPath),
                 messageIdFactory,
-                preInstanciationStage.getSessionProvider())))
+                preInstanciationStage.getSessionProvider(), new JsoupTextExtractor())))
             .noPreDeletionHooks()
             .storeQuotaManager()
             .build();
diff --git a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageFlagSearchTest.java b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageFlagSearchTest.java
index 76669104c2..eca27dae24 100644
--- a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageFlagSearchTest.java
+++ b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageFlagSearchTest.java
@@ -47,6 +47,7 @@ import org.apache.james.mailbox.model.MessageRange;
 import org.apache.james.mailbox.model.SearchQuery;
 import org.apache.james.mailbox.store.StoreMailboxManager;
 import org.apache.james.mailbox.store.StoreMessageManager;
+import org.apache.james.mailbox.store.extractor.JsoupTextExtractor;
 import org.apache.james.mailbox.store.search.MessageSearchIndex;
 import org.apache.james.utils.UpdatableTickingClock;
 import org.apache.lucene.document.Document;
@@ -156,7 +157,7 @@ class LuceneMailboxMessageFlagSearchTest {
                 .listeningSearchIndex(Throwing.function(preInstanciationStage -> new LuceneMessageSearchIndex(
                         preInstanciationStage.getMapperFactory(), new InMemoryId.Factory(), new ByteBuffersDirectory(),
                         new InMemoryMessageId.Factory(),
-                        preInstanciationStage.getSessionProvider())))
+                        preInstanciationStage.getSessionProvider(), new JsoupTextExtractor())))
                 .noPreDeletionHooks()
                 .storeQuotaManager()
                 .build();
diff --git a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java
index 58d82d02f7..fb2ff37c3c 100644
--- a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java
+++ b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java
@@ -52,6 +52,7 @@ import org.apache.james.mailbox.model.TestMessageId;
 import org.apache.james.mailbox.model.UidValidity;
 import org.apache.james.mailbox.model.UpdatedFlags;
 import org.apache.james.mailbox.store.MessageBuilder;
+import org.apache.james.mailbox.store.extractor.JsoupTextExtractor;
 import org.apache.james.mailbox.store.search.ListeningMessageSearchIndex;
 import org.apache.james.mailbox.store.search.ListeningMessageSearchIndexContract;
 import org.apache.lucene.store.ByteBuffersDirectory;
@@ -112,7 +113,7 @@ class LuceneMailboxMessageSearchIndexTest {
         id3 = factory.generate();
         id4 = factory.generate();
         id5 = factory.generate();
-        index = new LuceneMessageSearchIndex(null, new TestId.Factory(), new ByteBuffersDirectory(), true, factory, null);
+        index = new LuceneMessageSearchIndex(null, new TestId.Factory(), new ByteBuffersDirectory(), true, factory, null, new JsoupTextExtractor());
         index.setEnableSuffixMatch(true);
         Map<String, String> headersSubject = new HashMap<>();
         headersSubject.put("Subject", "test (fwd)");
diff --git a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMemorySearchHighLighterTest.java b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMemorySearchHighLighterTest.java
index ca97c12441..23af88f0b8 100644
--- a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMemorySearchHighLighterTest.java
+++ b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMemorySearchHighLighterTest.java
@@ -19,31 +19,45 @@
 package org.apache.james.mailbox.lucene.search;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
 
 import org.apache.james.core.Username;
+import org.apache.james.mailbox.MailboxManager;
 import org.apache.james.mailbox.MailboxSession;
 import org.apache.james.mailbox.MessageManager;
 import org.apache.james.mailbox.exception.MailboxException;
 import org.apache.james.mailbox.inmemory.InMemoryId;
 import org.apache.james.mailbox.inmemory.InMemoryMessageId;
 import org.apache.james.mailbox.inmemory.manager.InMemoryIntegrationResources;
+import org.apache.james.mailbox.model.ComposedMessageId;
 import org.apache.james.mailbox.model.Mailbox;
 import org.apache.james.mailbox.model.MailboxACL;
 import org.apache.james.mailbox.model.MailboxId;
 import org.apache.james.mailbox.model.MailboxPath;
 import org.apache.james.mailbox.model.MessageId;
+import org.apache.james.mailbox.model.MultimailboxesSearchQuery;
 import org.apache.james.mailbox.model.SearchQuery;
 import org.apache.james.mailbox.searchhighligt.SearchHighLighterContract;
 import org.apache.james.mailbox.searchhighligt.SearchHighlighter;
 import org.apache.james.mailbox.searchhighligt.SearchHighlighterConfiguration;
+import org.apache.james.mailbox.searchhighligt.SearchSnippet;
 import org.apache.james.mailbox.store.StoreMailboxManager;
 import org.apache.james.mailbox.store.StoreMessageManager;
+import org.apache.james.mailbox.store.extractor.JsoupTextExtractor;
 import org.apache.james.mailbox.store.search.MessageSearchIndex;
+import org.apache.james.mime4j.dom.Message;
+import org.apache.james.util.ClassLoaderUtils;
 import org.apache.lucene.store.ByteBuffersDirectory;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 
 import com.github.fge.lambdas.Throwing;
 
+import reactor.core.publisher.Flux;
+
 class LuceneMemorySearchHighLighterTest implements SearchHighLighterContract {
     private MessageSearchIndex messageSearchIndex;
     private StoreMailboxManager storeMailboxManager;
@@ -62,7 +76,7 @@ class LuceneMemorySearchHighLighterTest implements SearchHighLighterContract {
             .listeningSearchIndex(Throwing.function(preInstanciationStage -> new LuceneMessageSearchIndex(
                 preInstanciationStage.getMapperFactory(), new InMemoryId.Factory(), new ByteBuffersDirectory(),
                 messageIdFactory,
-                preInstanciationStage.getSessionProvider())))
+                preInstanciationStage.getSessionProvider(), new JsoupTextExtractor())))
             .noPreDeletionHooks()
             .storeQuotaManager()
             .build();
@@ -112,4 +126,46 @@ class LuceneMemorySearchHighLighterTest implements SearchHighLighterContract {
             MailboxACL.command().forUser(delegated).rights(MailboxACL.FULL_RIGHTS).asAddition(),
             session(owner))).run();
     }
+
+
+    @Test
+    void shouldHighlightAttachmentTextContentWhenTextBodyDoesNotMatch() throws Exception {
+        assumeTrue(storeMailboxManager.getSupportedSearchCapabilities().contains(MailboxManager.SearchCapabilities.Attachment));
+        MailboxSession session = session(USERNAME1);
+
+        ComposedMessageId m1 = appendMessage(MessageManager.AppendCommand.from(
+                Message.Builder.of()
+                    .setTo("to@james.local")
+                    .setSubject("Hallo, Thx Matthieu for your help")
+                    .setBody("append contentA to inbox", StandardCharsets.UTF_8)),
+            session).getId();
+
+        // m2 has an attachment with text content: "This is a beautiful banana"
+        ComposedMessageId m2 = inboxMessageManager.appendMessage(
+            MessageManager.AppendCommand.builder()
+                .build(ClassLoaderUtils.getSystemResourceAsSharedStream("eml/emailWithTextAttachment.eml")),
+            session).getId();
+
+        verifyMessageWasIndexed(2);
+
+        String keywordSearch = "beautiful";
+        MultimailboxesSearchQuery multiMailboxSearch = MultimailboxesSearchQuery.from(SearchQuery.of(
+                new SearchQuery.ConjunctionCriterion(SearchQuery.Conjunction.OR,
+                    List.of(SearchQuery.bodyContains(keywordSearch),
+                        SearchQuery.attachmentContains(keywordSearch)))))
+            .inMailboxes(List.of(m1.getMailboxId(), m2.getMailboxId()))
+            .build();
+
+        List<SearchSnippet> searchSnippets = Flux.from(testee().highlightSearch(List.of(m1.getMessageId(), m2.getMessageId()), multiMailboxSearch, session))
+            .collectList()
+            .block();
+
+        assertThat(searchSnippets).hasSize(1);
+
+        assertThat(searchSnippets.getFirst().highlightedBody())
+            .isPresent()
+            .satisfies(highlightedBody -> {
+                assertThat(highlightedBody.get()).contains("This is a <mark>beautiful</mark> banana");
+            });
+    }
 }
diff --git a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndexTest.java b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndexTest.java
index 8046ba8ca3..fcb8b2d347 100644
--- a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndexTest.java
+++ b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndexTest.java
@@ -19,8 +19,10 @@
 
 package org.apache.james.mailbox.lucene.search;
 
+import java.net.URISyntaxException;
 import java.util.List;
 
+import org.apache.james.mailbox.extractor.TextExtractor;
 import org.apache.james.mailbox.inmemory.InMemoryId;
 import org.apache.james.mailbox.inmemory.InMemoryMessageId;
 import org.apache.james.mailbox.inmemory.manager.InMemoryIntegrationResources;
@@ -28,13 +30,34 @@ import org.apache.james.mailbox.model.MailboxId;
 import org.apache.james.mailbox.model.MessageId;
 import org.apache.james.mailbox.model.SearchQuery;
 import org.apache.james.mailbox.store.search.AbstractMessageSearchIndexTest;
+import org.apache.james.mailbox.tika.TikaConfiguration;
+import org.apache.james.mailbox.tika.TikaExtension;
+import org.apache.james.mailbox.tika.TikaHttpClientImpl;
+import org.apache.james.mailbox.tika.TikaTextExtractor;
+import org.apache.james.metrics.tests.RecordingMetricFactory;
 import org.apache.lucene.store.ByteBuffersDirectory;
+import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
 
 import com.github.fge.lambdas.Throwing;
 
 class LuceneMessageSearchIndexTest extends AbstractMessageSearchIndexTest {
+    @RegisterExtension
+    static TikaExtension tika = new TikaExtension();
+
+    static TextExtractor textExtractor;
+
+    @BeforeAll
+    static void setUpClass() throws URISyntaxException {
+        textExtractor = new TikaTextExtractor(new RecordingMetricFactory(),
+            new TikaHttpClientImpl(TikaConfiguration.builder()
+                .host(tika.getIp())
+                .port(tika.getPort())
+                .timeoutInMillis(tika.getTimeoutInMillis())
+                .build()));
+    }
 
     @Override
     protected void awaitMessageCount(List<MailboxId> mailboxIds, SearchQuery query, long messageCount) {
@@ -51,7 +74,7 @@ class LuceneMessageSearchIndexTest extends AbstractMessageSearchIndexTest {
             .listeningSearchIndex(Throwing.function(preInstanciationStage -> new LuceneMessageSearchIndex(
                 preInstanciationStage.getMapperFactory(), new InMemoryId.Factory(), new ByteBuffersDirectory(),
                 new InMemoryMessageId.Factory(),
-                preInstanciationStage.getSessionProvider())))
+                preInstanciationStage.getSessionProvider(), textExtractor)))
             .noPreDeletionHooks()
             .storeQuotaManager()
             .build();
diff --git a/mpt/impl/imap-mailbox/lucenesearch/src/test/java/org/apache/james/mpt/imapmailbox/lucenesearch/host/LuceneSearchHostSystem.java b/mpt/impl/imap-mailbox/lucenesearch/src/test/java/org/apache/james/mpt/imapmailbox/lucenesearch/host/LuceneSearchHostSystem.java
index ee82414000..cc37e7db84 100644
--- a/mpt/impl/imap-mailbox/lucenesearch/src/test/java/org/apache/james/mpt/imapmailbox/lucenesearch/host/LuceneSearchHostSystem.java
+++ b/mpt/impl/imap-mailbox/lucenesearch/src/test/java/org/apache/james/mpt/imapmailbox/lucenesearch/host/LuceneSearchHostSystem.java
@@ -34,6 +34,7 @@ import org.apache.james.mailbox.inmemory.InMemoryMessageId;
 import org.apache.james.mailbox.inmemory.manager.InMemoryIntegrationResources;
 import org.apache.james.mailbox.lucene.search.LuceneMessageSearchIndex;
 import org.apache.james.mailbox.store.StoreSubscriptionManager;
+import org.apache.james.mailbox.store.extractor.DefaultTextExtractor;
 import org.apache.james.mailbox.store.quota.NoQuotaManager;
 import org.apache.james.metrics.logger.DefaultMetricFactory;
 import org.apache.james.mpt.api.ImapFeatures;
@@ -66,7 +67,7 @@ public class LuceneSearchHostSystem extends JamesImapHostSystem {
             .listeningSearchIndex(Throwing.function(preInstanciationStage -> new LuceneMessageSearchIndex(
                 preInstanciationStage.getMapperFactory(), new InMemoryId.Factory(), new ByteBuffersDirectory(),
                 new InMemoryMessageId.Factory(),
-                preInstanciationStage.getSessionProvider())))
+                preInstanciationStage.getSessionProvider(), new DefaultTextExtractor())))
             .noPreDeletionHooks()
             .storeQuotaManager()
             .build();


---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org