You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@james.apache.org by bt...@apache.org on 2022/05/04 01:10:58 UTC
[james-project] 03/12: [PERF] IMAP SEARCH: Reduce overhead for large responses
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit ddc7fd523d8912d545ce7efa0ad5abeb413e3354
Author: Loan <tl...@linagora.com>
AuthorDate: Tue Apr 26 10:27:04 2022 +0700
[PERF] IMAP SEARCH: Reduce overhead for large responses
- Limit Long boxing impact
- Avoid unneeded array copy
- Limit stream usage (local use, large overhead)
---
.../james/imap/encode/SearchResponseEncoder.java | 16 ++++++--
.../imap/message/response/SearchResponse.java | 15 ++++----
.../james/imap/processor/SearchProcessor.java | 44 ++++++++++------------
.../imap/encode/SearchResponseEncoderTest.java | 12 +++++-
.../james/imap/processor/SearchProcessorTest.java | 12 +++---
5 files changed, 56 insertions(+), 43 deletions(-)
diff --git a/protocols/imap/src/main/java/org/apache/james/imap/encode/SearchResponseEncoder.java b/protocols/imap/src/main/java/org/apache/james/imap/encode/SearchResponseEncoder.java
index 8f134abbc8..91f7de88cc 100644
--- a/protocols/imap/src/main/java/org/apache/james/imap/encode/SearchResponseEncoder.java
+++ b/protocols/imap/src/main/java/org/apache/james/imap/encode/SearchResponseEncoder.java
@@ -24,6 +24,9 @@ import org.apache.james.imap.api.ImapConstants;
import org.apache.james.imap.message.response.SearchResponse;
import org.apache.james.mailbox.ModSeq;
+import it.unimi.dsi.fastutil.longs.LongConsumer;
+import it.unimi.dsi.fastutil.longs.LongList;
+
/**
* Encoders IMAP4rev1 <code>SEARCH</code> responses.
*/
@@ -35,14 +38,19 @@ public class SearchResponseEncoder implements ImapResponseEncoder<SearchResponse
@Override
public void encode(SearchResponse response, ImapResponseComposer composer) throws IOException {
- final long[] ids = response.getIds();
+ LongList ids = response.getIds();
ModSeq highestModSeq = response.getHighestModSeq();
composer.untagged();
composer.commandName(ImapConstants.SEARCH_COMMAND);
if (ids != null) {
- for (long id : ids) {
- composer.message(id);
- }
+ LongConsumer longConsumer = l -> {
+ try {
+ composer.message(l);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ };
+ ids.forEach(longConsumer);
}
// add MODSEQ
diff --git a/protocols/imap/src/main/java/org/apache/james/imap/message/response/SearchResponse.java b/protocols/imap/src/main/java/org/apache/james/imap/message/response/SearchResponse.java
index 422bf41c36..1a82626a3f 100644
--- a/protocols/imap/src/main/java/org/apache/james/imap/message/response/SearchResponse.java
+++ b/protocols/imap/src/main/java/org/apache/james/imap/message/response/SearchResponse.java
@@ -19,17 +19,18 @@
package org.apache.james.imap.message.response;
-import java.util.Arrays;
import java.util.Objects;
import org.apache.james.imap.api.message.response.ImapResponseMessage;
import org.apache.james.mailbox.ModSeq;
+import it.unimi.dsi.fastutil.longs.LongList;
+
/**
* A <code>SEARCH</code> response.
*/
public class SearchResponse implements ImapResponseMessage {
- private final long[] ids;
+ private final LongList ids;
private final ModSeq highestModSeq;
/**
@@ -37,7 +38,7 @@ public class SearchResponse implements ImapResponseMessage {
*
* @param ids ids, not null
*/
- public SearchResponse(long[] ids, ModSeq highestModSeq) {
+ public SearchResponse(LongList ids, ModSeq highestModSeq) {
this.ids = ids;
this.highestModSeq = highestModSeq;
}
@@ -47,7 +48,7 @@ public class SearchResponse implements ImapResponseMessage {
*
* @return the ids, not null
*/
- public final long[] getIds() {
+ public final LongList getIds() {
return ids;
}
@@ -66,7 +67,7 @@ public class SearchResponse implements ImapResponseMessage {
if (o instanceof SearchResponse) {
SearchResponse that = (SearchResponse) o;
- return Arrays.equals(this.ids, that.ids)
+ return Objects.equals(this.ids, that.ids)
&& Objects.equals(this.highestModSeq, that.highestModSeq);
}
return false;
@@ -74,7 +75,7 @@ public class SearchResponse implements ImapResponseMessage {
@Override
public final int hashCode() {
- return Objects.hash(Arrays.hashCode(ids), highestModSeq);
+ return Objects.hash(ids, highestModSeq);
}
/**
@@ -88,7 +89,7 @@ public class SearchResponse implements ImapResponseMessage {
StringBuilder retValue = new StringBuilder();
- retValue.append("SearchResponse ( ").append("ids = ").append(Arrays.toString(this.ids)).append(TAB).append(" )");
+ retValue.append("SearchResponse ( ").append("ids = ").append(this.ids).append(TAB).append(" )");
return retValue.toString();
}
diff --git a/protocols/imap/src/main/java/org/apache/james/imap/processor/SearchProcessor.java b/protocols/imap/src/main/java/org/apache/james/imap/processor/SearchProcessor.java
index 73c44d0c5a..a7821017b1 100644
--- a/protocols/imap/src/main/java/org/apache/james/imap/processor/SearchProcessor.java
+++ b/protocols/imap/src/main/java/org/apache/james/imap/processor/SearchProcessor.java
@@ -27,7 +27,6 @@ import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Optional;
-import java.util.stream.Stream;
import javax.mail.Flags.Flag;
@@ -71,6 +70,8 @@ import org.slf4j.LoggerFactory;
import com.github.fge.lambdas.Throwing;
import com.google.common.collect.ImmutableList;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import it.unimi.dsi.fastutil.longs.LongList;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
@@ -146,8 +147,7 @@ public class SearchProcessor extends AbstractMailboxProcessor<SearchRequest> imp
}
private ImapResponseMessage toResponse(SearchRequest request, ImapSession session, Collection<MessageUid> uids, Optional<ModSeq> highestModSeq) {
- Collection<Long> results = asResults(session, request.isUseUids(), uids);
- long[] ids = toArray(results);
+ LongList ids = asResults(session, request.isUseUids(), uids);
List<SearchResultOption> resultOptions = request.getSearchOperation().getResultOptions();
if (resultOptions == null || resultOptions.isEmpty()) {
@@ -157,9 +157,9 @@ public class SearchProcessor extends AbstractMailboxProcessor<SearchRequest> imp
}
}
- private ImapResponseMessage handleResultOptions(SearchRequest request, ImapSession session, Collection<MessageUid> uids, ModSeq highestModSeq, long[] ids) {
+ private ImapResponseMessage handleResultOptions(SearchRequest request, ImapSession session, Collection<MessageUid> uids, ModSeq highestModSeq, LongList ids) {
List<SearchResultOption> resultOptions = request.getSearchOperation().getResultOptions();
- List<Long> idList = new ArrayList<>(ids.length);
+ List<Long> idList = new ArrayList<>(ids.size());
for (long id : ids) {
idList.add(id);
}
@@ -187,11 +187,11 @@ public class SearchProcessor extends AbstractMailboxProcessor<SearchRequest> imp
if (esearch) {
long min = -1;
long max = -1;
- long count = ids.length;
+ long count = ids.size();
- if (ids.length > 0) {
- min = ids[0];
- max = ids[ids.length - 1];
+ if (ids.size() > 0) {
+ min = ids.getLong(0);
+ max = ids.getLong(ids.size() - 1);
}
@@ -221,20 +221,20 @@ public class SearchProcessor extends AbstractMailboxProcessor<SearchRequest> imp
}
}
- private Collection<Long> asResults(ImapSession session, boolean useUids, Collection<MessageUid> uids) {
+ private LongList asResults(ImapSession session, boolean useUids, Collection<MessageUid> uids) {
+ LongList result = new LongArrayList(uids.size());
+ // Avoid using streams here as the overhead for large search responses is massive.
if (useUids) {
- return uids.stream()
- .map(MessageUid::asLong)
- .collect(ImmutableList.toImmutableList());
+ for (MessageUid uid: uids) {
+ result.add(uid.asLong());
+ }
} else {
- return uids.stream()
- .map(uid -> session.getSelected().msn(uid))
- .flatMap(Throwing.function(nullableMsn ->
- nullableMsn.fold(
- Stream::empty,
- msn -> Stream.of(Integer.valueOf(msn.asInt()).longValue()))))
- .collect(ImmutableList.toImmutableList());
+ for (MessageUid uid: uids) {
+ session.getSelected().msn(uid)
+ .ifPresent(l -> result.add(l.asInt()));
+ }
}
+ return result;
}
private Mono<Collection<MessageUid>> performUidSearch(MessageManager mailbox, SearchQuery query, MailboxSession msession) throws MailboxException {
@@ -242,10 +242,6 @@ public class SearchProcessor extends AbstractMailboxProcessor<SearchRequest> imp
.collect(ImmutableList.toImmutableList());
}
- private long[] toArray(Collection<Long> results) {
- return results.stream().mapToLong(x -> x).toArray();
- }
-
/**
* Find the highest mod-sequence number in the given {@link MessageRange}'s.
*/
diff --git a/protocols/imap/src/test/java/org/apache/james/imap/encode/SearchResponseEncoderTest.java b/protocols/imap/src/test/java/org/apache/james/imap/encode/SearchResponseEncoderTest.java
index 9d821f0f0b..5fd1135337 100644
--- a/protocols/imap/src/test/java/org/apache/james/imap/encode/SearchResponseEncoderTest.java
+++ b/protocols/imap/src/test/java/org/apache/james/imap/encode/SearchResponseEncoderTest.java
@@ -27,8 +27,18 @@ import org.apache.james.imap.message.response.SearchResponse;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import it.unimi.dsi.fastutil.longs.LongList;
+
class SearchResponseEncoderTest {
- private static final long[] IDS = { 1, 4, 9, 16 };
+ private static final LongList IDS = new LongArrayList();
+
+ static {
+ IDS.add(1L);
+ IDS.add(4L);
+ IDS.add(9L);
+ IDS.add(16L);
+ }
private SearchResponse response;
private SearchResponseEncoder encoder;
diff --git a/protocols/imap/src/test/java/org/apache/james/imap/processor/SearchProcessorTest.java b/protocols/imap/src/test/java/org/apache/james/imap/processor/SearchProcessorTest.java
index 487ad8d724..a1416f4339 100644
--- a/protocols/imap/src/test/java/org/apache/james/imap/processor/SearchProcessorTest.java
+++ b/protocols/imap/src/test/java/org/apache/james/imap/processor/SearchProcessorTest.java
@@ -20,12 +20,8 @@
package org.apache.james.imap.processor;
import static org.apache.james.imap.ImapFixture.TAG;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.ArgumentMatchers.same;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
import java.util.ArrayList;
import java.util.Calendar;
@@ -72,6 +68,8 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.stubbing.Answer;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import it.unimi.dsi.fastutil.longs.LongList;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
@@ -89,7 +87,7 @@ public class SearchProcessorTest {
private static final String KEYWORD = "BD3";
- private static final long[] EMPTY = {};
+ private static final LongList EMPTY = new LongArrayList();
private static final String ADDRESS = "John Smith <jo...@example.org>";
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org