You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@james.apache.org by bt...@apache.org on 2023/04/27 10:55:14 UTC
[james-project] branch master updated (32c42b631d -> 315f30114c)
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
from 32c42b631d JAMES-3903 Only archive the code coverage report when the build is success.
new 64d394f721 [FIX] Strengthen IMAP ESEARCH tests
new f812548421 [PERF] Optimize IMAP ESEARCH options
new 315f30114c [PERF] IMAP ESEARCH: working with uids is not needed
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../imap/scripts/SearchOptionAggregations.test | 62 ++++++++++++++++++++--
.../james/imap/encode/ESearchResponseEncoder.java | 8 +--
.../imap/message/response/ESearchResponse.java | 9 +---
.../james/imap/processor/SearchProcessor.java | 61 +++++++++++++--------
4 files changed, 100 insertions(+), 40 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org
[james-project] 01/03: [FIX] Strengthen IMAP ESEARCH tests
Posted by bt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 64d394f721b6296bd5387226dc8a6a5445847fa2
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Wed Apr 26 17:28:41 2023 +0700
[FIX] Strengthen IMAP ESEARCH tests
---
.../imap/scripts/SearchOptionAggregations.test | 62 ++++++++++++++++++++--
1 file changed, 59 insertions(+), 3 deletions(-)
diff --git a/mpt/impl/imap-mailbox/core/src/main/resources/org/apache/james/imap/scripts/SearchOptionAggregations.test b/mpt/impl/imap-mailbox/core/src/main/resources/org/apache/james/imap/scripts/SearchOptionAggregations.test
index c408a3f4f4..79ebebf289 100644
--- a/mpt/impl/imap-mailbox/core/src/main/resources/org/apache/james/imap/scripts/SearchOptionAggregations.test
+++ b/mpt/impl/imap-mailbox/core/src/main/resources/org/apache/james/imap/scripts/SearchOptionAggregations.test
@@ -143,6 +143,62 @@ S: \* 5 FETCH .*
}
S: i OK FETCH completed.
-C: h SEARCH RETURN (ALL) FLAGGED
-S: \* ESEARCH \(TAG "h"\) ALL 1:2,4:5
-S: h OK SEARCH completed.
\ No newline at end of file
+C: j SEARCH RETURN (ALL) FLAGGED
+S: \* ESEARCH \(TAG "j"\) ALL 1:2,4:5
+S: j OK SEARCH completed.
+
+C: k UID SEARCH RETURN (ALL) FLAGGED
+S: \* ESEARCH \(TAG "k"\) UID ALL 1:2,4:5
+S: k OK SEARCH completed.
+
+C: l UID SEARCH RETURN (MIN MAX COUNT) FLAGGED
+S: \* ESEARCH \(TAG "l"\) UID MIN 1 MAX 5 COUNT 4
+S: l OK SEARCH completed.
+
+C: m UID SEARCH RETURN (ALL SAVE) FLAGGED
+S: \* ESEARCH \(TAG "m"\) UID ALL 1:2,4:5
+S: m OK SEARCH completed.
+
+C: n FETCH $ FLAGS
+SUB {
+S: \* 1 FETCH .*
+S: \* 2 FETCH .*
+S: \* 4 FETCH .*
+S: \* 5 FETCH .*
+}
+S: n OK FETCH completed.
+
+C: o UID SEARCH RETURN (MIN MAX SAVE) FLAGGED
+S: \* ESEARCH \(TAG "o"\) UID MIN 1 MAX 5
+S: o OK SEARCH completed.
+
+C: p FETCH $ FLAGS
+SUB {
+S: \* 1 FETCH .*
+S: \* 5 FETCH .*
+}
+S: p OK FETCH completed.
+
+C: q UID SEARCH RETURN (MIN MAX COUNT SAVE) FLAGGED
+S: \* ESEARCH \(TAG "q"\) UID MIN 1 MAX 5 COUNT 4
+S: q OK SEARCH completed.
+
+C: r FETCH $ FLAGS
+SUB {
+S: \* 1 FETCH .*
+S: \* 2 FETCH .*
+S: \* 4 FETCH .*
+S: \* 5 FETCH .*
+}
+S: r OK FETCH completed.
+
+C: s UID SEARCH RETURN (MIN MAX) ANSWERED
+S: \* ESEARCH \(TAG "s"\) UID
+S: s OK SEARCH completed.
+
+C: t UID SEARCH RETURN (MIN MAX SAVE) ANSWERED
+S: \* ESEARCH \(TAG "t"\) UID
+S: t OK SEARCH completed.
+
+C: u FETCH $ FLAGS
+S: u OK FETCH completed.
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org
[james-project] 02/03: [PERF] Optimize IMAP ESEARCH options
Posted by bt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit f812548421384d44f0d2f0a580c249a2db957b30
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Wed Apr 26 17:34:31 2023 +0700
[PERF] Optimize IMAP ESEARCH options
- Compute UID ranges only if needed
- Compute the idSet directly from the LongList to work in-place and drop complexity
---
.../james/imap/processor/SearchProcessor.java | 69 ++++++++++++++++------
1 file changed, 50 insertions(+), 19 deletions(-)
diff --git a/protocols/imap/src/main/java/org/apache/james/imap/processor/SearchProcessor.java b/protocols/imap/src/main/java/org/apache/james/imap/processor/SearchProcessor.java
index 6d506900c3..499047dd94 100644
--- a/protocols/imap/src/main/java/org/apache/james/imap/processor/SearchProcessor.java
+++ b/protocols/imap/src/main/java/org/apache/james/imap/processor/SearchProcessor.java
@@ -22,6 +22,7 @@ package org.apache.james.imap.processor;
import static org.apache.james.mailbox.MessageManager.MailboxMetaData.RecentMode.IGNORE;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
@@ -74,6 +75,8 @@ import com.github.fge.lambdas.Throwing;
import com.google.common.collect.ImmutableList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
+import it.unimi.dsi.fastutil.longs.LongComparators;
+import it.unimi.dsi.fastutil.longs.LongConsumer;
import it.unimi.dsi.fastutil.longs.LongList;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
@@ -164,22 +167,8 @@ public class SearchProcessor extends AbstractMailboxProcessor<SearchRequest> imp
private ImapResponseMessage handleResultOptions(SearchRequest request, ImapSession session, Collection<MessageUid> uids, ModSeq highestModSeq, LongList ids) {
List<SearchResultOption> resultOptions = request.getSearchOperation().getResultOptions();
- List<Long> idList = new ArrayList<>(ids.size());
- for (long id : ids) {
- idList.add(id);
- }
- List<IdRange> idsAsRanges = new ArrayList<>();
- for (Long id: idList) {
- idsAsRanges.add(new IdRange(id));
- }
- IdRange[] idRanges = IdRange.mergeRanges(idsAsRanges).toArray(IdRange[]::new);
-
- List<UidRange> uidsAsRanges = new ArrayList<>();
- for (MessageUid uid: uids) {
- uidsAsRanges.add(new UidRange(uid));
- }
- UidRange[] uidRanges = UidRange.mergeRanges(uidsAsRanges).toArray(UidRange[]::new);
+ IdRange[] idRanges = asRanges(ids);
boolean esearch = false;
for (SearchResultOption resultOption : resultOptions) {
@@ -194,12 +183,11 @@ public class SearchProcessor extends AbstractMailboxProcessor<SearchRequest> imp
long max = -1;
long count = ids.size();
- if (ids.size() > 0) {
+ if (!ids.isEmpty()) {
min = ids.getLong(0);
max = ids.getLong(ids.size() - 1);
}
-
// Save the sequence-set for later usage. This is part of SEARCHRES
if (resultOptions.contains(SearchResultOption.SAVE)) {
if (resultOptions.contains(SearchResultOption.ALL) || resultOptions.contains(SearchResultOption.COUNT)) {
@@ -207,17 +195,18 @@ public class SearchProcessor extends AbstractMailboxProcessor<SearchRequest> imp
SearchResUtil.saveSequenceSet(session, idRanges);
} else {
List<IdRange> savedRanges = new ArrayList<>();
- if (resultOptions.contains(SearchResultOption.MIN)) {
+ if (resultOptions.contains(SearchResultOption.MIN) && min > 0) {
// Store the MIN
savedRanges.add(new IdRange(min));
}
- if (resultOptions.contains(SearchResultOption.MAX)) {
+ if (resultOptions.contains(SearchResultOption.MAX) && max > 0) {
// Store the MAX
savedRanges.add(new IdRange(max));
}
SearchResUtil.saveSequenceSet(session, savedRanges.toArray(IdRange[]::new));
}
}
+ UidRange[] uidRanges = uidRangeIfNeeded(request, resultOptions, uids);
return new ESearchResponse(min, max, count, idRanges, uidRanges, highestModSeq, request.getTag(), request.isUseUids(), resultOptions);
} else {
// Just save the returned sequence-set as this is not SEARCHRES + ESEARCH
@@ -226,6 +215,48 @@ public class SearchProcessor extends AbstractMailboxProcessor<SearchRequest> imp
}
}
+ private UidRange[] uidRangeIfNeeded(SearchRequest request, List<SearchResultOption> resultOptions, Collection<MessageUid> uids) {
+ if (resultOptions.contains(SearchResultOption.ALL) && request.isUseUids()) {
+ List<UidRange> uidsAsRanges = new ArrayList<>();
+ for (MessageUid uid: uids) {
+ uidsAsRanges.add(new UidRange(uid));
+ }
+ return UidRange.mergeRanges(uidsAsRanges).toArray(UidRange[]::new);
+ }
+ return null;
+ }
+
+ /**
+ * Optimization of IdRange.mergeRanges(idsAsRanges) for list of long
+ */
+ private IdRange[] asRanges(LongList ids) {
+ ids.sort(LongComparators.NATURAL_COMPARATOR);
+ List<IdRange> idsAsRanges = new ArrayList<>();
+ long lowBound = -1;
+ long highBound = -1;
+ for (int i = 0; i < ids.size(); i++) {
+ long id = ids.getLong(i);
+ // Initialize
+ if (lowBound == -1) {
+ lowBound = id;
+ highBound = id;
+ continue;
+ }
+ if (id == highBound + 1) {
+ highBound = id;
+ continue;
+ }
+ idsAsRanges.add(new IdRange(lowBound, highBound));
+ lowBound = id;
+ highBound = id;
+ }
+ if (lowBound != -1 && highBound != -1) {
+ idsAsRanges.add(new IdRange(lowBound, highBound));
+ }
+ IdRange[] result = new IdRange[idsAsRanges.size()];
+ return idsAsRanges.toArray(result);
+ }
+
private LongList asResults(ImapSession session, boolean useUids, Collection<MessageUid> uids) {
LongList result = new LongArrayList(uids.size());
// Avoid using streams here as the overhead for large search responses is massive.
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org
[james-project] 03/03: [PERF] IMAP ESEARCH: working with uids is not needed
Posted by bt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 315f30114cbdbe9b99015436e7a1028c5b46eb20
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Wed Apr 26 17:38:49 2023 +0700
[PERF] IMAP ESEARCH: working with uids is not needed
The untyped idSet represent the exact same data
---
.../james/imap/encode/ESearchResponseEncoder.java | 8 +-------
.../james/imap/message/response/ESearchResponse.java | 9 +--------
.../apache/james/imap/processor/SearchProcessor.java | 20 +++-----------------
3 files changed, 5 insertions(+), 32 deletions(-)
diff --git a/protocols/imap/src/main/java/org/apache/james/imap/encode/ESearchResponseEncoder.java b/protocols/imap/src/main/java/org/apache/james/imap/encode/ESearchResponseEncoder.java
index 819090fa9d..212758f1d5 100644
--- a/protocols/imap/src/main/java/org/apache/james/imap/encode/ESearchResponseEncoder.java
+++ b/protocols/imap/src/main/java/org/apache/james/imap/encode/ESearchResponseEncoder.java
@@ -24,7 +24,6 @@ import java.util.List;
import org.apache.james.imap.api.ImapConstants;
import org.apache.james.imap.api.Tag;
import org.apache.james.imap.api.message.IdRange;
-import org.apache.james.imap.api.message.UidRange;
import org.apache.james.imap.api.message.request.SearchResultOption;
import org.apache.james.imap.message.response.ESearchResponse;
import org.apache.james.mailbox.ModSeq;
@@ -45,7 +44,6 @@ public class ESearchResponseEncoder implements ImapResponseEncoder<ESearchRespon
long max = response.getMaxUid();
long count = response.getCount();
IdRange[] all = response.getAll();
- UidRange[] allUids = response.getAllUids();
boolean useUid = response.getUseUid();
ModSeq highestModSeq = response.getHighestModSeq();
List<SearchResultOption> options = response.getSearchResultOptions();
@@ -63,14 +61,10 @@ public class ESearchResponseEncoder implements ImapResponseEncoder<ESearchRespon
if (options.contains(SearchResultOption.COUNT)) {
composer.message(SearchResultOption.COUNT.name()).message(count);
}
- if (!useUid && all != null && all.length > 0 && options.contains(SearchResultOption.ALL)) {
+ if (all != null && all.length > 0 && options.contains(SearchResultOption.ALL)) {
composer.message(SearchResultOption.ALL.name());
composer.sequenceSet(all);
}
- if (useUid && allUids != null && allUids.length > 0 && options.contains(SearchResultOption.ALL)) {
- composer.message(SearchResultOption.ALL.name());
- composer.sequenceSet(allUids);
- }
// Add the MODSEQ to the response if needed.
//
diff --git a/protocols/imap/src/main/java/org/apache/james/imap/message/response/ESearchResponse.java b/protocols/imap/src/main/java/org/apache/james/imap/message/response/ESearchResponse.java
index 89e0437fdb..b21ad10f82 100644
--- a/protocols/imap/src/main/java/org/apache/james/imap/message/response/ESearchResponse.java
+++ b/protocols/imap/src/main/java/org/apache/james/imap/message/response/ESearchResponse.java
@@ -23,7 +23,6 @@ import java.util.List;
import org.apache.james.imap.api.Tag;
import org.apache.james.imap.api.message.IdRange;
-import org.apache.james.imap.api.message.UidRange;
import org.apache.james.imap.api.message.request.SearchResultOption;
import org.apache.james.imap.api.message.response.ImapResponseMessage;
import org.apache.james.mailbox.ModSeq;
@@ -37,16 +36,14 @@ public class ESearchResponse implements ImapResponseMessage {
private final boolean useUid;
private final List<SearchResultOption> options;
private final ModSeq highestModSeq;
- private UidRange[] allUids;
- public ESearchResponse(long minUid, long maxUid, long count, IdRange[] all, UidRange[] allUids, ModSeq highestModSeq, Tag tag, boolean useUid, List<SearchResultOption> options) {
+ public ESearchResponse(long minUid, long maxUid, long count, IdRange[] all, ModSeq highestModSeq, Tag tag, boolean useUid, List<SearchResultOption> options) {
this.options = options;
this.minUid = minUid;
this.maxUid = maxUid;
this.count = count;
this.all = all;
this.tag = tag;
- this.allUids = allUids;
this.useUid = useUid;
this.highestModSeq = highestModSeq;
}
@@ -66,10 +63,6 @@ public class ESearchResponse implements ImapResponseMessage {
public IdRange[] getAll() {
return all;
}
-
- public UidRange[] getAllUids() {
- return allUids;
- }
public Tag getTag() {
return tag;
diff --git a/protocols/imap/src/main/java/org/apache/james/imap/processor/SearchProcessor.java b/protocols/imap/src/main/java/org/apache/james/imap/processor/SearchProcessor.java
index 499047dd94..3b33d0abc9 100644
--- a/protocols/imap/src/main/java/org/apache/james/imap/processor/SearchProcessor.java
+++ b/protocols/imap/src/main/java/org/apache/james/imap/processor/SearchProcessor.java
@@ -22,7 +22,6 @@ package org.apache.james.imap.processor;
import static org.apache.james.mailbox.MessageManager.MailboxMetaData.RecentMode.IGNORE;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
@@ -76,7 +75,6 @@ import com.google.common.collect.ImmutableList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongComparators;
-import it.unimi.dsi.fastutil.longs.LongConsumer;
import it.unimi.dsi.fastutil.longs.LongList;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
@@ -161,11 +159,11 @@ public class SearchProcessor extends AbstractMailboxProcessor<SearchRequest> imp
if (resultOptions == null || resultOptions.isEmpty()) {
return new SearchResponse(ids, highestModSeq.orElse(null));
} else {
- return handleResultOptions(request, session, uids, highestModSeq.orElse(null), ids);
+ return handleResultOptions(request, session, highestModSeq.orElse(null), ids);
}
}
- private ImapResponseMessage handleResultOptions(SearchRequest request, ImapSession session, Collection<MessageUid> uids, ModSeq highestModSeq, LongList ids) {
+ private ImapResponseMessage handleResultOptions(SearchRequest request, ImapSession session, ModSeq highestModSeq, LongList ids) {
List<SearchResultOption> resultOptions = request.getSearchOperation().getResultOptions();
IdRange[] idRanges = asRanges(ids);
@@ -206,8 +204,7 @@ public class SearchProcessor extends AbstractMailboxProcessor<SearchRequest> imp
SearchResUtil.saveSequenceSet(session, savedRanges.toArray(IdRange[]::new));
}
}
- UidRange[] uidRanges = uidRangeIfNeeded(request, resultOptions, uids);
- return new ESearchResponse(min, max, count, idRanges, uidRanges, highestModSeq, request.getTag(), request.isUseUids(), resultOptions);
+ return new ESearchResponse(min, max, count, idRanges, highestModSeq, request.getTag(), request.isUseUids(), resultOptions);
} else {
// Just save the returned sequence-set as this is not SEARCHRES + ESEARCH
SearchResUtil.saveSequenceSet(session, idRanges);
@@ -215,17 +212,6 @@ public class SearchProcessor extends AbstractMailboxProcessor<SearchRequest> imp
}
}
- private UidRange[] uidRangeIfNeeded(SearchRequest request, List<SearchResultOption> resultOptions, Collection<MessageUid> uids) {
- if (resultOptions.contains(SearchResultOption.ALL) && request.isUseUids()) {
- List<UidRange> uidsAsRanges = new ArrayList<>();
- for (MessageUid uid: uids) {
- uidsAsRanges.add(new UidRange(uid));
- }
- return UidRange.mergeRanges(uidsAsRanges).toArray(UidRange[]::new);
- }
- return null;
- }
-
/**
* Optimization of IdRange.mergeRanges(idsAsRanges) for list of long
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org