You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2018/10/15 06:28:37 UTC
[14/17] james-project git commit: JAMES-2555 Simplify reIndexing: no
more needed message event tracking
JAMES-2555 Simplify reIndexing: no more needed message event tracking
Because we reindex a list of UID for a given mailbox, if:
- A new message arrive it will be handled by the normal indexing process
- A concurrently deleted message will be filtered out as it can not be retrieved from storage
- And we will read the latest applied flag value
Note that global event tracking is still needed for mailbox path renames.
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/9ff8a260
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/9ff8a260
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/9ff8a260
Branch: refs/heads/master
Commit: 9ff8a260ea27993c46eb660c64a8881d9cf9f4a8
Parents: f354e3c
Author: Benoit Tellier <bt...@linagora.com>
Authored: Fri Oct 12 16:59:03 2018 +0700
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Mon Oct 15 13:18:20 2018 +0700
----------------------------------------------------------------------
.../tools/indexer/ReIndexerPerformer.java | 33 +----
.../indexer/events/ImpactingMessageEvent.java | 144 -------------------
.../registrations/MailboxRegistration.java | 70 ---------
.../registrations/MailboxRegistrationTest.java | 107 --------------
4 files changed, 2 insertions(+), 352 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/9ff8a260/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java
----------------------------------------------------------------------
diff --git a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java
index c2d471d..ce88882 100644
--- a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java
+++ b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java
@@ -38,15 +38,11 @@ import org.apache.james.mailbox.store.search.ListeningMessageSearchIndex;
import org.apache.james.task.Task;
import org.apache.james.util.OptionalUtils;
import org.apache.james.util.streams.Iterators;
-import org.apache.mailbox.tools.indexer.events.ImpactingEventType;
-import org.apache.mailbox.tools.indexer.events.ImpactingMessageEvent;
import org.apache.mailbox.tools.indexer.registrations.GlobalRegistration;
-import org.apache.mailbox.tools.indexer.registrations.MailboxRegistration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.github.fge.lambdas.Throwing;
-import com.google.common.collect.Lists;
class ReIndexerPerformer {
@@ -105,34 +101,27 @@ class ReIndexerPerformer {
}
private Task.Result reIndex(MailboxPath path, MailboxSession mailboxSession, ReprocessingContext reprocessingContext) throws MailboxException {
- MailboxRegistration mailboxRegistration = new MailboxRegistration(path);
LOGGER.info("Intend to reindex {}", path.asString());
Mailbox mailbox = mailboxSessionMapperFactory.getMailboxMapper(mailboxSession).findMailboxByPath(path);
messageSearchIndex.deleteAll(mailboxSession, mailbox);
- mailboxManager.addListener(path, mailboxRegistration, mailboxSession);
try {
return Iterators.toStream(
mailboxSessionMapperFactory.getMessageMapper(mailboxSession)
.findInMailbox(mailbox, MessageRange.all(), MessageMapper.FetchType.Metadata, NO_LIMIT))
.map(MailboxMessage::getUid)
- .map(uid -> handleMessageReIndexing(mailboxSession, mailboxRegistration, mailbox, uid))
+ .map(uid -> handleMessageReIndexing(mailboxSession, mailbox, uid))
.peek(reprocessingContext::updateAccordingToReprocessingResult)
.reduce(Task::combine)
.orElse(Task.Result.COMPLETED);
} finally {
LOGGER.info("Finish to reindex {}", path.asString());
- mailboxManager.removeListener(path, mailboxRegistration, mailboxSession);
}
}
- private Task.Result handleMessageReIndexing(MailboxSession mailboxSession, MailboxRegistration mailboxRegistration, Mailbox mailbox, MessageUid uid) {
+ private Task.Result handleMessageReIndexing(MailboxSession mailboxSession, Mailbox mailbox, MessageUid uid) {
try {
- Optional<ImpactingMessageEvent> impactingMessageEvent = findMostRelevant(mailboxRegistration.getImpactingEvents(uid));
-
Optional.of(uid)
- .filter(x -> !wasDeleted(impactingMessageEvent))
.flatMap(Throwing.function(mUid -> fullyReadMessage(mailboxSession, mailbox, mUid)))
- .map(message -> messageUpdateRegardingEvents(message, impactingMessageEvent))
.ifPresent(Throwing.consumer(message -> messageSearchIndex.add(mailboxSession, mailbox, message)));
return Task.Result.COMPLETED;
} catch (Exception e) {
@@ -141,24 +130,6 @@ class ReIndexerPerformer {
}
}
- private Optional<ImpactingMessageEvent> findMostRelevant(List<ImpactingMessageEvent> messageEvents) {
- for (ImpactingMessageEvent impactingMessageEvent : messageEvents) {
- if (impactingMessageEvent.getType().equals(ImpactingEventType.Deletion)) {
- return Optional.of(impactingMessageEvent);
- }
- }
- return Lists.reverse(messageEvents).stream().findFirst();
- }
-
- private boolean wasDeleted(Optional<ImpactingMessageEvent> impactingMessageEvent) {
- return impactingMessageEvent.map(ImpactingMessageEvent::wasDeleted).orElse(false);
- }
-
- private MailboxMessage messageUpdateRegardingEvents(MailboxMessage message, Optional<ImpactingMessageEvent> impactingMessageEvent) {
- impactingMessageEvent.flatMap(ImpactingMessageEvent::newFlags).ifPresent(message::setFlags);
- return message;
- }
-
private Optional<MailboxMessage> fullyReadMessage(MailboxSession mailboxSession, Mailbox mailbox, MessageUid mUid) throws MailboxException {
return Iterators.toStream(mailboxSessionMapperFactory.getMessageMapper(mailboxSession)
.findInMailbox(mailbox, MessageRange.one(mUid), MessageMapper.FetchType.Full, SINGLE_MESSAGE))
http://git-wip-us.apache.org/repos/asf/james-project/blob/9ff8a260/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/events/ImpactingMessageEvent.java
----------------------------------------------------------------------
diff --git a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/events/ImpactingMessageEvent.java b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/events/ImpactingMessageEvent.java
deleted file mode 100644
index 527f463..0000000
--- a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/events/ImpactingMessageEvent.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/****************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one *
- * or more contributor license agreements. See the NOTICE file *
- * distributed with this work for additional information *
- * regarding copyright ownership. The ASF licenses this file *
- * to you under the Apache License, Version 2.0 (the *
- * "License"); you may not use this file except in compliance *
- * with the License. You may obtain a copy of the License at *
- * *
- * http://www.apache.org/licenses/LICENSE-2.0 *
- * *
- * Unless required by applicable law or agreed to in writing, *
- * software distributed under the License is distributed on an *
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
- * KIND, either express or implied. See the License for the *
- * specific language governing permissions and limitations *
- * under the License. *
- ****************************************************************/
-
-package org.apache.mailbox.tools.indexer.events;
-
-import java.util.Objects;
-import java.util.Optional;
-
-import javax.mail.Flags;
-
-import org.apache.james.mailbox.MessageUid;
-import org.apache.james.mailbox.model.MailboxPath;
-
-public interface ImpactingMessageEvent extends ImpactingEvent {
-
- MessageUid getUid();
-
- Optional<Flags> newFlags();
-
- boolean wasDeleted();
-
- class FlagsMessageEvent implements ImpactingMessageEvent {
-
- private final MailboxPath mailboxPath;
- private final MessageUid uid;
- private final Flags flags;
-
- public FlagsMessageEvent(MailboxPath mailboxPath, MessageUid uid, Flags flags) {
- this.mailboxPath = mailboxPath;
- this.uid = uid;
- this.flags = flags;
- }
-
- @Override
- public MessageUid getUid() {
- return uid;
- }
-
- @Override
- public MailboxPath getMailboxPath() {
- return mailboxPath;
- }
-
- @Override
- public ImpactingEventType getType() {
- return ImpactingEventType.FlagsUpdate;
- }
-
- @Override
- public Optional<Flags> newFlags() {
- return Optional.of(flags);
- }
-
- @Override
- public boolean wasDeleted() {
- return false;
- }
-
- @Override
- public final boolean equals(Object o) {
- if (o instanceof FlagsMessageEvent) {
- FlagsMessageEvent that = (FlagsMessageEvent) o;
-
- return Objects.equals(this.mailboxPath, that.mailboxPath)
- && Objects.equals(this.uid, that.uid)
- && Objects.equals(this.flags, that.flags);
- }
- return false;
- }
-
- @Override
- public final int hashCode() {
- return Objects.hash(mailboxPath, uid, flags);
- }
- }
-
- class MessageDeletedEvent implements ImpactingMessageEvent {
-
- private final MailboxPath mailboxPath;
- private final MessageUid uid;
-
- public MessageDeletedEvent(MailboxPath mailboxPath, MessageUid uid) {
- this.mailboxPath = mailboxPath;
- this.uid = uid;
- }
-
- @Override
- public MessageUid getUid() {
- return uid;
- }
-
- @Override
- public MailboxPath getMailboxPath() {
- return mailboxPath;
- }
-
- @Override
- public ImpactingEventType getType() {
- return ImpactingEventType.Deletion;
- }
-
- @Override
- public Optional<Flags> newFlags() {
- return Optional.empty();
- }
-
- @Override
- public boolean wasDeleted() {
- return true;
- }
-
- @Override
- public final boolean equals(Object o) {
- if (o instanceof MessageDeletedEvent) {
- MessageDeletedEvent that = (MessageDeletedEvent) o;
-
- return Objects.equals(this.mailboxPath, that.mailboxPath)
- && Objects.equals(this.uid, that.uid);
- }
- return false;
- }
-
- @Override
- public final int hashCode() {
- return Objects.hash(mailboxPath, uid);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/james-project/blob/9ff8a260/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistration.java
----------------------------------------------------------------------
diff --git a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistration.java b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistration.java
deleted file mode 100644
index 251fde8..0000000
--- a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistration.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/****************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one *
- * or more contributor license agreements. See the NOTICE file *
- * distributed with this work for additional information *
- * regarding copyright ownership. The ASF licenses this file *
- * to you under the Apache License, Version 2.0 (the *
- * "License"); you may not use this file except in compliance *
- * with the License. You may obtain a copy of the License at *
- * *
- * http://www.apache.org/licenses/LICENSE-2.0 *
- * *
- * Unless required by applicable law or agreed to in writing, *
- * software distributed under the License is distributed on an *
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
- * KIND, either express or implied. See the License for the *
- * specific language governing permissions and limitations *
- * under the License. *
- ****************************************************************/
-
-package org.apache.mailbox.tools.indexer.registrations;
-
-import java.util.List;
-
-import org.apache.james.mailbox.Event;
-import org.apache.james.mailbox.MailboxListener;
-import org.apache.james.mailbox.MessageUid;
-import org.apache.james.mailbox.model.MailboxPath;
-import org.apache.james.mailbox.model.UpdatedFlags;
-import org.apache.mailbox.tools.indexer.events.ImpactingMessageEvent;
-import org.apache.mailbox.tools.indexer.events.ImpactingMessageEvent.FlagsMessageEvent;
-import org.apache.mailbox.tools.indexer.events.ImpactingMessageEvent.MessageDeletedEvent;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Multimaps;
-
-public class MailboxRegistration implements MailboxListener {
-
- private final Multimap<MessageUid, ImpactingMessageEvent> impactingMessageEvents;
- private final MailboxPath mailboxPath;
-
- public MailboxRegistration(MailboxPath mailboxPath) {
- this.impactingMessageEvents = Multimaps.synchronizedMultimap(ArrayListMultimap.<MessageUid, ImpactingMessageEvent>create());
- this.mailboxPath = mailboxPath;
- }
-
- @Override
- public ListenerType getType() {
- return ListenerType.MAILBOX;
- }
-
- public List<ImpactingMessageEvent> getImpactingEvents(MessageUid uid) {
- return ImmutableList.copyOf(impactingMessageEvents.get(uid));
- }
-
- @Override
- public void event(Event event) {
- if (event instanceof FlagsUpdated) {
- for (UpdatedFlags updatedFlags : ((FlagsUpdated) event).getUpdatedFlags()) {
- impactingMessageEvents.put(updatedFlags.getUid(), new FlagsMessageEvent(mailboxPath, updatedFlags.getUid(), updatedFlags.getNewFlags()));
- }
- } else if (event instanceof Expunged) {
- for (MessageUid uid: ((Expunged) event).getUids()) {
- impactingMessageEvents.put(uid, new MessageDeletedEvent(mailboxPath, uid));
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/james-project/blob/9ff8a260/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistrationTest.java
----------------------------------------------------------------------
diff --git a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistrationTest.java b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistrationTest.java
deleted file mode 100644
index 3b9bad2..0000000
--- a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistrationTest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/****************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one *
- * or more contributor license agreements. See the NOTICE file *
- * distributed with this work for additional information *
- * regarding copyright ownership. The ASF licenses this file *
- * to you under the Apache License, Version 2.0 (the *
- * "License"); you may not use this file except in compliance *
- * with the License. You may obtain a copy of the License at *
- * *
- * http://www.apache.org/licenses/LICENSE-2.0 *
- * *
- * Unless required by applicable law or agreed to in writing, *
- * software distributed under the License is distributed on an *
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
- * KIND, either express or implied. See the License for the *
- * specific language governing permissions and limitations *
- * under the License. *
- ****************************************************************/
-
-package org.apache.mailbox.tools.indexer.registrations;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-import java.util.Date;
-import java.util.TreeMap;
-
-import javax.mail.Flags;
-
-import org.apache.james.mailbox.MailboxListener;
-import org.apache.james.mailbox.MessageUid;
-import org.apache.james.mailbox.mock.MockMailboxSession;
-import org.apache.james.mailbox.model.MailboxPath;
-import org.apache.james.mailbox.model.MessageMetaData;
-import org.apache.james.mailbox.model.UpdatedFlags;
-import org.apache.james.mailbox.store.SimpleMessageMetaData;
-import org.apache.james.mailbox.store.event.EventFactory;
-import org.apache.james.mailbox.store.mail.model.DefaultMessageId;
-import org.apache.james.mailbox.store.mail.model.MailboxMessage;
-import org.apache.james.mailbox.store.mail.model.impl.SimpleMailbox;
-import org.apache.mailbox.tools.indexer.events.ImpactingMessageEvent.FlagsMessageEvent;
-import org.apache.mailbox.tools.indexer.events.ImpactingMessageEvent.MessageDeletedEvent;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-
-public class MailboxRegistrationTest {
-
- public static final MailboxPath INBOX = MailboxPath.forUser("btellier@apache.org", "INBOX");
- private static final MessageUid UID = MessageUid.of(18);
- private static final int UID_VALIDITY = 45;
- private static final SimpleMailbox MAILBOX = new SimpleMailbox(INBOX, UID_VALIDITY);
- private static final int MOD_SEQ = 21;
- private static final int SIZE = 41;
- private static final Flags NEW_FLAGS = new Flags(Flags.Flag.ANSWERED);
- private MailboxRegistration mailboxRegistration;
- private EventFactory eventFactory;
- private MockMailboxSession session;
-
- @BeforeEach
- void setUp() {
- session = new MockMailboxSession("test");
- eventFactory = new EventFactory();
- mailboxRegistration = new MailboxRegistration(INBOX);
- }
-
- @Test
- void reportedEventsShouldBeInitiallyEmpty() {
- assertThat(mailboxRegistration.getImpactingEvents(UID)).isEmpty();
- }
-
-
- @Test
- void addedEventsShouldNotBeReported() {
- TreeMap<MessageUid, MessageMetaData> treeMap = new TreeMap<>();
- treeMap.put(UID, new SimpleMessageMetaData(UID, MOD_SEQ, new Flags(), SIZE, new Date(), new DefaultMessageId()));
- MailboxListener.MailboxEvent event = eventFactory.added(session, treeMap, MAILBOX, ImmutableMap.<MessageUid, MailboxMessage>of());
- mailboxRegistration.event(event);
- assertThat(mailboxRegistration.getImpactingEvents(UID)).isEmpty();
- }
-
- @Test
- void expungedEventsShouldBeReported() {
- TreeMap<MessageUid, MessageMetaData> treeMap = new TreeMap<>();
- treeMap.put(UID, new SimpleMessageMetaData(UID, MOD_SEQ, new Flags(), SIZE, new Date(), new DefaultMessageId()));
- MailboxListener.MailboxEvent event = eventFactory.expunged(session, treeMap, MAILBOX);
- mailboxRegistration.event(event);
- assertThat(mailboxRegistration.getImpactingEvents(UID)).containsExactly(new MessageDeletedEvent(INBOX, UID));
- }
-
- @Test
- void flagsEventsShouldBeReported() {
- MailboxListener.MailboxEvent event = eventFactory.flagsUpdated(session,
- Lists.newArrayList(UID),
- MAILBOX,
- Lists.newArrayList(UpdatedFlags.builder()
- .uid(UID)
- .modSeq(MOD_SEQ)
- .oldFlags(new Flags())
- .newFlags(NEW_FLAGS)
- .build()));
- mailboxRegistration.event(event);
- assertThat(mailboxRegistration.getImpactingEvents(UID)).containsExactly(new FlagsMessageEvent(INBOX, UID, NEW_FLAGS));
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org