You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@james.apache.org by bt...@apache.org on 2026/02/06 13:12:31 UTC
(james-project) branch master updated: JAMES-4166 Implement collapseThreads support for Lucene (#2935)
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
The following commit(s) were added to refs/heads/master by this push:
new 955c610e88 JAMES-4166 Implement collapseThreads support for Lucene (#2935)
955c610e88 is described below
commit 955c610e88cdb7e7d91e0d64e69167c78bf1420d
Author: Trần Hồng Quân <55...@users.noreply.github.com>
AuthorDate: Fri Feb 6 20:12:25 2026 +0700
JAMES-4166 Implement collapseThreads support for Lucene (#2935)
---
mailbox/lucene/pom.xml | 5 ++
.../lucene/search/LuceneIndexableDocument.java | 6 +-
.../lucene/search/LuceneMessageSearchIndex.java | 70 ++++++++++++++++++----
.../search/LuceneMessageSearchIndexTest.java | 5 ++
.../memory/MemoryEmailQueryMethodNoViewTest.java | 12 ----
.../rfc8621/memory/MemoryEmailQueryMethodTest.java | 12 ----
upgrade-instructions.md | 13 ++++
7 files changed, 86 insertions(+), 37 deletions(-)
diff --git a/mailbox/lucene/pom.xml b/mailbox/lucene/pom.xml
index 95df49120f..9a1784159a 100644
--- a/mailbox/lucene/pom.xml
+++ b/mailbox/lucene/pom.xml
@@ -122,6 +122,11 @@
<artifactId>lucene-core</artifactId>
<version>${lucene.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-grouping</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-highlighter</artifactId>
diff --git a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneIndexableDocument.java b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneIndexableDocument.java
index 4f14d14c3f..1f45d594a1 100644
--- a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneIndexableDocument.java
+++ b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneIndexableDocument.java
@@ -88,6 +88,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
@@ -143,7 +144,10 @@ public class LuceneIndexableDocument {
Optional.ofNullable(SearchUtil.getSerializedMessageIdIfSupportedByUnderlyingStorageOrNull(message))
.ifPresent(serializedMessageId -> doc.add(new StringField(MESSAGE_ID_FIELD, serializedMessageId, Field.Store.YES)));
Optional.ofNullable(SearchUtil.getSerializedThreadIdIfSupportedByUnderlyingStorageOrNull(message))
- .ifPresent(serializedThreadId -> doc.add(new StringField(THREAD_ID_FIELD, serializedThreadId, Field.Store.YES)));
+ .ifPresent(serializedThreadId -> {
+ doc.add(new StringField(THREAD_ID_FIELD, serializedThreadId, Field.Store.YES));
+ doc.add(new SortedDocValuesField(THREAD_ID_FIELD, new BytesRef(serializedThreadId)));
+ });
HeaderCollection headerCollection = mimePartExtracted.getHeaderCollection();
diff --git a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
index 60fc932240..b9298d7013 100644
--- a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
+++ b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java
@@ -131,7 +131,12 @@ import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.search.grouping.GroupDocs;
+import org.apache.lucene.search.grouping.GroupingSearch;
+import org.apache.lucene.search.grouping.TermGroupSelector;
+import org.apache.lucene.search.grouping.TopGroups;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -308,6 +313,13 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
return Flux.empty();
}
+ if (searchQuery.shouldCollapseThreads()) {
+ return searchCollapseThreads(mailboxIds, searchQuery, searchOptions);
+ }
+ return searchWithoutCollapseThreads(mailboxIds, searchQuery, searchOptions);
+ }
+
+ private Flux<MessageId> searchWithoutCollapseThreads(Collection<MailboxId> mailboxIds, SearchQuery searchQuery, SearchOptions searchOptions) throws MailboxException {
long requestedLimit = Math.addExact(searchOptions.offset().getOffset(), searchOptions.limit().getLimit().orElseThrow());
return Flux.fromIterable(searchMultimap(mailboxIds, searchQuery)
@@ -320,6 +332,36 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
.collect(ImmutableList.toImmutableList()));
}
+ private Flux<MessageId> searchCollapseThreads(Collection<MailboxId> mailboxIds, SearchQuery searchQuery, SearchOptions searchOptions) throws MailboxException {
+ Query query = buildQuery(mailboxIds, searchQuery);
+
+ try (IndexReader reader = DirectoryReader.open(writer)) {
+ IndexSearcher searcher = new IndexSearcher(reader);
+
+ GroupingSearch groupingSearch = new GroupingSearch(new TermGroupSelector(THREAD_ID_FIELD));
+ Sort sort = createSort(searchQuery.getSorts());
+ groupingSearch.setGroupSort(sort);
+ groupingSearch.setSortWithinGroup(sort);
+ // get the first message of each thread group
+ groupingSearch.setGroupDocsOffset(0);
+ groupingSearch.setGroupDocsLimit(1);
+
+ int groupOffset = Math.toIntExact(searchOptions.offset().getOffset());
+ int topNGroups = Math.toIntExact(searchOptions.limit().getLimit().orElseThrow());
+
+ TopGroups<BytesRef> topGroups = groupingSearch.search(searcher, query, groupOffset, topNGroups);
+ List<MessageId> result = new ArrayList<>(topGroups.groups.length);
+ for (GroupDocs<BytesRef> group : topGroups.groups) {
+ ScoreDoc[] scoreDocs = group.scoreDocs();
+ Document document = searcher.storedFields().document(scoreDocs[0].doc);
+ documentToSearchResult(document).getMessageId().ifPresent(result::add);
+ }
+ return Flux.fromIterable(result);
+ } catch (IOException e) {
+ throw new MailboxException("Unable to search the mailbox", e);
+ }
+ }
+
private List<SearchResult> searchMultimap(Collection<MailboxId> mailboxIds, SearchQuery searchQuery) throws MailboxException {
return searchDocument(mailboxIds, searchQuery, maxQueryResults)
.stream()
@@ -336,22 +378,12 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
}
public List<Document> searchDocument(Collection<MailboxId> mailboxIds, SearchQuery searchQuery, int maxQueryResults) throws MailboxException {
- Query inMailboxes = buildQueryFromMailboxes(mailboxIds);
-
try (IndexReader reader = DirectoryReader.open(writer)) {
IndexSearcher searcher = new IndexSearcher(reader);
- BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
- queryBuilder.add(inMailboxes, BooleanClause.Occur.MUST);
- // Not return flags documents
- queryBuilder.add(new PrefixQuery(new Term(FLAGS_FIELD, "")), BooleanClause.Occur.MUST_NOT);
-
- List<Criterion> crits = searchQuery.getCriteria();
- for (Criterion crit : crits) {
- queryBuilder.add(createQuery(crit, inMailboxes, searchQuery.getRecentMessageUids()), BooleanClause.Occur.MUST);
- }
+ Query query = buildQuery(mailboxIds, searchQuery);
// query for all the documents sorted as specified in the SearchQuery
- TopDocs docs = searcher.search(queryBuilder.build(), maxQueryResults, createSort(searchQuery.getSorts()));
+ TopDocs docs = searcher.search(query, maxQueryResults, createSort(searchQuery.getSorts()));
return Stream.of(docs.scoreDocs)
.map(Throwing.function(sDoc -> searcher.storedFields().document(sDoc.doc)))
@@ -370,6 +402,20 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex {
return queryBuilder.build();
}
+ private Query buildQuery(Collection<MailboxId> mailboxIds, SearchQuery searchQuery) throws MailboxException {
+ Query inMailboxes = buildQueryFromMailboxes(mailboxIds);
+ BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
+ queryBuilder.add(inMailboxes, BooleanClause.Occur.MUST);
+ // Not return flags documents
+ queryBuilder.add(new PrefixQuery(new Term(FLAGS_FIELD, "")), BooleanClause.Occur.MUST_NOT);
+
+ List<Criterion> crits = searchQuery.getCriteria();
+ for (Criterion crit : crits) {
+ queryBuilder.add(createQuery(crit, inMailboxes, searchQuery.getRecentMessageUids()), BooleanClause.Occur.MUST);
+ }
+ return queryBuilder.build();
+ }
+
private String toSentDateField(DateResolution res) {
return switch (res) {
case Year -> SENT_DATE_FIELD_YEAR_RESOLUTION;
diff --git a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndexTest.java b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndexTest.java
index fcb8b2d347..1c59cb4d93 100644
--- a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndexTest.java
+++ b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndexTest.java
@@ -96,6 +96,11 @@ class LuceneMessageSearchIndexTest extends AbstractMessageSearchIndexTest {
return InMemoryMessageId.of(1000);
}
+ @Override
+ protected boolean supportsCollapseThreads() {
+ return true;
+ }
+
@Disabled("JAMES-1799: ignoring failing test after generalizing OpenSearch test suite to other mailbox search backends")
@Override
public void uidShouldreturnEveryThing() {
diff --git a/server/protocols/jmap-rfc-8621-integration-tests/memory-jmap-rfc-8621-integration-tests/src/test/java/org/apache/james/jmap/rfc8621/memory/MemoryEmailQueryMethodNoViewTest.java b/server/protocols/jmap-rfc-8621-integration-tests/memory-jmap-rfc-8621-integration-tests/src/test/java/org/apache/james/jmap/rfc8621/memory/MemoryEmailQueryMethodNoViewTest.java
index 2587a843ef..49141a8c7b 100644
--- a/server/protocols/jmap-rfc-8621-integration-tests/memory-jmap-rfc-8621-integration-tests/src/test/java/org/apache/james/jmap/rfc8621/memory/MemoryEmailQueryMethodNoViewTest.java
+++ b/server/protocols/jmap-rfc-8621-integration-tests/memory-jmap-rfc-8621-integration-tests/src/test/java/org/apache/james/jmap/rfc8621/memory/MemoryEmailQueryMethodNoViewTest.java
@@ -102,16 +102,4 @@ public class MemoryEmailQueryMethodNoViewTest implements EmailQueryMethodContrac
public void inMailboxBeforeSortedByReceivedAtShouldCollapseThreads(GuiceJamesServer server) {
}
- @Test
- @Override
- @Disabled("JAMES-4166 collapseThreads does not support Lucene implementation yet")
- public void collapseThreadsShouldApplyOnSearchIndexPath(GuiceJamesServer server) {
- }
-
- @Test
- @Override
- @Disabled("JAMES-4166 collapseThreads does not support Lucene implementation yet")
- public void collapseThreadsShouldApplyPaginationOnCollapsedResults(GuiceJamesServer server) {
- }
-
}
diff --git a/server/protocols/jmap-rfc-8621-integration-tests/memory-jmap-rfc-8621-integration-tests/src/test/java/org/apache/james/jmap/rfc8621/memory/MemoryEmailQueryMethodTest.java b/server/protocols/jmap-rfc-8621-integration-tests/memory-jmap-rfc-8621-integration-tests/src/test/java/org/apache/james/jmap/rfc8621/memory/MemoryEmailQueryMethodTest.java
index 970107e19c..f5152dd222 100644
--- a/server/protocols/jmap-rfc-8621-integration-tests/memory-jmap-rfc-8621-integration-tests/src/test/java/org/apache/james/jmap/rfc8621/memory/MemoryEmailQueryMethodTest.java
+++ b/server/protocols/jmap-rfc-8621-integration-tests/memory-jmap-rfc-8621-integration-tests/src/test/java/org/apache/james/jmap/rfc8621/memory/MemoryEmailQueryMethodTest.java
@@ -52,16 +52,4 @@ public class MemoryEmailQueryMethodTest extends MemoryBase implements EmailQuery
EmailQueryMethodContract.super.shouldListMailsReceivedAfterADate(server);
}
- @Test
- @Override
- @Disabled("JAMES-4166 collapseThreads does not support Lucene implementation yet")
- public void collapseThreadsShouldApplyOnSearchIndexPath(GuiceJamesServer server) {
- }
-
- @Test
- @Override
- @Disabled("JAMES-4166 collapseThreads does not support Lucene implementation yet")
- public void collapseThreadsShouldApplyPaginationOnCollapsedResults(GuiceJamesServer server) {
- }
-
}
diff --git a/upgrade-instructions.md b/upgrade-instructions.md
index 145b04d820..cc79b2954e 100644
--- a/upgrade-instructions.md
+++ b/upgrade-instructions.md
@@ -17,6 +17,19 @@ Changes to apply between 3.9.x and 3.10.0 will be reported here.
Change list:
- [Adding thread_id column to Cassandra email_query_view_sent_at and email_query_view_received_at tables](#adding-thread_id-column-to-cassandra-email_query_view_sent_at-and-email_query_view_received_at-tables)
- [Adding thread_id column to Postgresql email_query_view table](#adding-thread_id-column-to-postgresql-email_query_view-table)
+ - [Lucene mailbox index schema update for collapseThreads support](#lucene-mailbox-index-schema-update-for-collapsethreads-support)
+
+### Lucene mailbox index schema update for collapseThreads support
+
+Date: 06/02/2026
+
+Concerned products: James apps relying on Lucene as the search index
+
+JIRA: https://issues.apache.org/jira/browse/JAMES-4166
+
+James now requires the `threadId` field to be indexed as a SortedDocValuesField to support `collapseThreads` on mailbox search.
+
+After upgrading, you need to rebuild the Lucene mailbox index by [reindexing all mails](https://james.apache.org/server/manage-webadmin.html#ReIndexing_all_mails).
### Adding thread_id column to Cassandra email_query_view_sent_at and email_query_view_received_at tables
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org