You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@james.apache.org by hq...@apache.org on 2026/08/18 07:47:27 UTC

(james-project) branch 3.9.x updated (da8f0a7e71 -> f70bfa3e59)

This is an automated email from the ASF dual-hosted git repository.

quantranhong1999 pushed a change to branch 3.9.x
in repository https://gitbox.apache.org/repos/asf/james-project.git


    from da8f0a7e71 [FIX] Prevent potentially blocking calls upon uploads (#3116) - backport 3.9.x
     new 67701826a7 [FIX] Handle missing Task history in Cassandra implem like PG
     new e5ac3864fb [FIX] Cassandra event store: include the snapshot in the event batch
     new 7762f6085e [FIX] Task cleanup should purge stalled tasks
     new b81c247365 [FIX] Allow canceling tasks with truncated history
     new ec435cd521 [FIX] Single row to read eventStore snapshots
     new d3f6ef0b08 [FIX] Toletate failures for Task additional information
     new f70bfa3e59 [FIX] ClearMailRepository should count only once

The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../eventstore/cassandra/CassandraEventStore.scala |  1 +
 .../eventstore/cassandra/EventStoreDao.scala       | 17 ++--
 .../cassandra/CassandraEventStoreExtension.scala   | 12 ++-
 .../cassandra/CassandraEventStoreTest.scala        | 19 +++++
 .../mailrepository/file/FileMailRepository.java    | 11 ++-
 .../mailrepository/jpa/JPAMailRepository.java      | 17 ++++
 .../postgres/PostgresMailRepository.java           |  8 +-
 .../postgres/PostgresMailRepositoryContentDAO.java |  5 +-
 .../james/mailrepository/api/MailRepository.java   | 15 ++++
 .../mailrepository/MailRepositoryContract.java     | 22 +++++
 .../blob/BlobMailRepositoryTest.java               |  8 ++
 .../cassandra/CassandraMailRepository.java         |  9 +-
 .../memory/MemoryMailRepository.java               | 10 +++
 .../webadmin/services/TasksCleanupService.java     | 47 ++++++++++-
 .../webadmin/routes/TasksCleanupRoutesTest.java    | 86 ++++++++++++++++++-
 .../webadmin/service/ClearMailRepositoryTask.java  | 32 +++++--
 .../service/ClearMailRepositoryTaskTest.java       | 97 +++++++++++++++++++++
 .../distributed/RabbitMQWorkQueue.java             |  3 +
 .../apache/james/task/SerialTaskManagerWorker.java | 20 ++++-
 .../james/task/eventsourcing/CommandHandlers.scala | 10 ++-
 .../task/eventsourcing/WorkerStatusListener.scala  | 34 ++++++--
 .../EventSourcingTaskManagerTest.java              | 98 +++++++++++++++++++++-
 22 files changed, 539 insertions(+), 42 deletions(-)


---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org


(james-project) 07/07: [FIX] ClearMailRepository should count only once

Posted by hq...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

quantranhong1999 pushed a commit to branch 3.9.x
in repository https://gitbox.apache.org/repos/asf/james-project.git

commit f70bfa3e599fe5381c1be410d1b3de3a0c964097
Author: Benoit TELLIER <bt...@linagora.com>
AuthorDate: Sun Aug 16 23:39:21 2026 +0700

    [FIX] ClearMailRepository should count only once
---
 .../mailrepository/file/FileMailRepository.java    | 11 ++-
 .../mailrepository/jpa/JPAMailRepository.java      | 17 ++++
 .../postgres/PostgresMailRepository.java           |  8 +-
 .../postgres/PostgresMailRepositoryContentDAO.java |  5 +-
 .../james/mailrepository/api/MailRepository.java   | 15 ++++
 .../mailrepository/MailRepositoryContract.java     | 22 +++++
 .../blob/BlobMailRepositoryTest.java               |  8 ++
 .../cassandra/CassandraMailRepository.java         |  9 +-
 .../memory/MemoryMailRepository.java               | 10 +++
 .../webadmin/service/ClearMailRepositoryTask.java  | 32 +++++--
 .../service/ClearMailRepositoryTaskTest.java       | 97 ++++++++++++++++++++++
 11 files changed, 220 insertions(+), 14 deletions(-)

diff --git a/server/data/data-file/src/main/java/org/apache/james/mailrepository/file/FileMailRepository.java b/server/data/data-file/src/main/java/org/apache/james/mailrepository/file/FileMailRepository.java
index a53371d291..b2e89daa5c 100644
--- a/server/data/data-file/src/main/java/org/apache/james/mailrepository/file/FileMailRepository.java
+++ b/server/data/data-file/src/main/java/org/apache/james/mailrepository/file/FileMailRepository.java
@@ -27,6 +27,7 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Optional;
+import java.util.function.Consumer;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -293,8 +294,16 @@ public class FileMailRepository implements MailRepository, Configurable, Initial
 
     @Override
     public void removeAll() {
+        removeAll(any -> { });
+    }
+
+    @Override
+    public void removeAll(Consumer<MailKey> progressCallback) {
         listStream()
-            .forEach(Throwing.<MailKey>consumer(this::remove).sneakyThrow());
+            .forEach(Throwing.<MailKey>consumer(key -> {
+                remove(key);
+                progressCallback.accept(key);
+            }).sneakyThrow());
     }
 
     private void internalRemove(MailKey key) {
diff --git a/server/data/data-jpa/src/main/java/org/apache/james/mailrepository/jpa/JPAMailRepository.java b/server/data/data-jpa/src/main/java/org/apache/james/mailrepository/jpa/JPAMailRepository.java
index dc35e4fd72..4426013eff 100644
--- a/server/data/data-jpa/src/main/java/org/apache/james/mailrepository/jpa/JPAMailRepository.java
+++ b/server/data/data-jpa/src/main/java/org/apache/james/mailrepository/jpa/JPAMailRepository.java
@@ -30,6 +30,7 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.StringTokenizer;
+import java.util.function.Consumer;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -77,6 +78,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.github.fge.lambdas.Throwing;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.UnmodifiableIterator;
 
 /**
  * Implementation of a MailRepository on a database via JPA.
@@ -84,6 +86,7 @@ import com.google.common.collect.ImmutableMap;
 public class JPAMailRepository implements MailRepository, Configurable, Initializable {
     private static final Logger LOGGER = LoggerFactory.getLogger(JPAMailRepository.class);
     private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+    private static final int DELETION_BATCH_SIZE = 100;
 
     private String repositoryName;
 
@@ -377,6 +380,20 @@ public class JPAMailRepository implements MailRepository, Configurable, Initiali
         }
     }
 
+    /**
+     * Deletes by batches rather than in one go, so that the caller gets to observe the progress being made.
+     */
+    @Override
+    public void removeAll(Consumer<MailKey> progressCallback) throws MessagingException {
+        UnmodifiableIterator<List<MailKey>> batches = com.google.common.collect.Iterators.partition(list(), DELETION_BATCH_SIZE);
+
+        while (batches.hasNext()) {
+            List<MailKey> batch = batches.next();
+            remove(batch);
+            batch.forEach(progressCallback);
+        }
+    }
+
     @Override
     public void removeAll() throws MessagingException {
         EntityManager entityManager = entityManager();
diff --git a/server/data/data-postgres/src/main/java/org/apache/james/mailrepository/postgres/PostgresMailRepository.java b/server/data/data-postgres/src/main/java/org/apache/james/mailrepository/postgres/PostgresMailRepository.java
index cc640377a1..510df2783e 100644
--- a/server/data/data-postgres/src/main/java/org/apache/james/mailrepository/postgres/PostgresMailRepository.java
+++ b/server/data/data-postgres/src/main/java/org/apache/james/mailrepository/postgres/PostgresMailRepository.java
@@ -21,6 +21,7 @@ package org.apache.james.mailrepository.postgres;
 
 import java.util.Collection;
 import java.util.Iterator;
+import java.util.function.Consumer;
 
 import jakarta.inject.Inject;
 import jakarta.mail.MessagingException;
@@ -80,6 +81,11 @@ public class PostgresMailRepository implements MailRepository {
 
     @Override
     public void removeAll() {
-        postgresMailRepositoryContentDAO.removeAll(url);
+        postgresMailRepositoryContentDAO.removeAll(url, any -> { });
+    }
+
+    @Override
+    public void removeAll(Consumer<MailKey> progressCallback) {
+        postgresMailRepositoryContentDAO.removeAll(url, progressCallback);
     }
 }
diff --git a/server/data/data-postgres/src/main/java/org/apache/james/mailrepository/postgres/PostgresMailRepositoryContentDAO.java b/server/data/data-postgres/src/main/java/org/apache/james/mailrepository/postgres/PostgresMailRepositoryContentDAO.java
index c90b0392f6..4524e5dc67 100644
--- a/server/data/data-postgres/src/main/java/org/apache/james/mailrepository/postgres/PostgresMailRepositoryContentDAO.java
+++ b/server/data/data-postgres/src/main/java/org/apache/james/mailrepository/postgres/PostgresMailRepositoryContentDAO.java
@@ -339,9 +339,10 @@ public class PostgresMailRepositoryContentDAO {
             .block();
     }
 
-    public void removeAll(MailRepositoryUrl url) {
+    public void removeAll(MailRepositoryUrl url, Consumer<MailKey> progressCallback) {
         listMailKeys(url)
-            .flatMap(mailKey -> removeReactive(mailKey, url), DEFAULT_CONCURRENCY)
+            .flatMap(mailKey -> removeReactive(mailKey, url).thenReturn(mailKey), DEFAULT_CONCURRENCY)
+            .doOnNext(progressCallback)
             .then()
             .block();
     }
diff --git a/server/mailrepository/mailrepository-api/src/main/java/org/apache/james/mailrepository/api/MailRepository.java b/server/mailrepository/mailrepository-api/src/main/java/org/apache/james/mailrepository/api/MailRepository.java
index 7013a5e893..ed1f55b508 100644
--- a/server/mailrepository/mailrepository-api/src/main/java/org/apache/james/mailrepository/api/MailRepository.java
+++ b/server/mailrepository/mailrepository-api/src/main/java/org/apache/james/mailrepository/api/MailRepository.java
@@ -22,6 +22,7 @@ package org.apache.james.mailrepository.api;
 import java.time.Instant;
 import java.util.Collection;
 import java.util.Iterator;
+import java.util.function.Consumer;
 import java.util.function.Predicate;
 
 import jakarta.mail.MessagingException;
@@ -212,4 +213,18 @@ public interface MailRepository {
      */
     void removeAll() throws MessagingException;
 
+    /**
+     * Removes all mails from this repository, notifying the caller of the mails being removed.
+     *
+     * Implementations deleting mails one by one should override this in order to report their progress: this
+     * allows long running deletions to be monitored without counting the remaining mails over and over, which
+     * is a costly operation on large repositories.
+     *
+     * Implementations relying on a bulk deletion can keep the default: the caller then only learns about the
+     * progress once the deletion completed.
+     */
+    default void removeAll(Consumer<MailKey> progressCallback) throws MessagingException {
+        removeAll();
+    }
+
 }
diff --git a/server/mailrepository/mailrepository-api/src/test/java/org/apache/james/mailrepository/MailRepositoryContract.java b/server/mailrepository/mailrepository-api/src/test/java/org/apache/james/mailrepository/MailRepositoryContract.java
index 6561119efc..8ff5f5763f 100644
--- a/server/mailrepository/mailrepository-api/src/test/java/org/apache/james/mailrepository/MailRepositoryContract.java
+++ b/server/mailrepository/mailrepository-api/src/test/java/org/apache/james/mailrepository/MailRepositoryContract.java
@@ -269,6 +269,28 @@ public interface MailRepositoryContract {
         testee.removeAll();
     }
 
+    @Test
+    default void removeAllShouldReportTheRemovedMails() throws Exception {
+        MailRepository testee = retrieveRepository();
+        testee.store(createMail(MAIL_1));
+        testee.store(createMail(MAIL_2));
+
+        ImmutableList.Builder<MailKey> removed = ImmutableList.builder();
+        testee.removeAll(removed::add);
+
+        assertThat(removed.build()).containsExactlyInAnyOrder(MAIL_1, MAIL_2);
+    }
+
+    @Test
+    default void removeAllShouldReportNothingWhenEmpty() throws Exception {
+        MailRepository testee = retrieveRepository();
+
+        ImmutableList.Builder<MailKey> removed = ImmutableList.builder();
+        testee.removeAll(removed::add);
+
+        assertThat(removed.build()).isEmpty();
+    }
+
     @Test
     default void retrieveShouldGetStoredEmojiMail() throws Exception {
         MailRepository testee = retrieveRepository();
diff --git a/server/mailrepository/mailrepository-blob/src/test/java/org/apache/james/mailrepository/blob/BlobMailRepositoryTest.java b/server/mailrepository/mailrepository-blob/src/test/java/org/apache/james/mailrepository/blob/BlobMailRepositoryTest.java
index fbf0f8036e..72adeae0e1 100644
--- a/server/mailrepository/mailrepository-blob/src/test/java/org/apache/james/mailrepository/blob/BlobMailRepositoryTest.java
+++ b/server/mailrepository/mailrepository-blob/src/test/java/org/apache/james/mailrepository/blob/BlobMailRepositoryTest.java
@@ -30,6 +30,8 @@ import org.apache.james.mailrepository.api.MailRepositoryUrl;
 import org.apache.james.mailrepository.api.Protocol;
 import org.jetbrains.annotations.NotNull;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
 
 class BlobMailRepositoryTest implements MailRepositoryContract {
 
@@ -67,4 +69,10 @@ class BlobMailRepositoryTest implements MailRepositoryContract {
     public MailRepository retrieveRepository(MailRepositoryPath path) {
         return buildBlobMailRepository(path);
     }
+
+    @Test
+    @Disabled("This repository removes blobs rather than mails: it can not tell which of them back a mail")
+    @Override
+    public void removeAllShouldReportTheRemovedMails() {
+    }
 }
\ No newline at end of file
diff --git a/server/mailrepository/mailrepository-cassandra/src/main/java/org/apache/james/mailrepository/cassandra/CassandraMailRepository.java b/server/mailrepository/mailrepository-cassandra/src/main/java/org/apache/james/mailrepository/cassandra/CassandraMailRepository.java
index 8051f1122c..15dbe63d64 100644
--- a/server/mailrepository/mailrepository-cassandra/src/main/java/org/apache/james/mailrepository/cassandra/CassandraMailRepository.java
+++ b/server/mailrepository/mailrepository-cassandra/src/main/java/org/apache/james/mailrepository/cassandra/CassandraMailRepository.java
@@ -24,6 +24,7 @@ import static org.apache.james.util.ReactorUtils.publishIfPresent;
 
 import java.util.Iterator;
 import java.util.Optional;
+import java.util.function.Consumer;
 
 import jakarta.inject.Inject;
 import jakarta.mail.MessagingException;
@@ -178,8 +179,14 @@ public class CassandraMailRepository implements MailRepository {
 
     @Override
     public void removeAll() {
+        removeAll(any -> { });
+    }
+
+    @Override
+    public void removeAll(Consumer<MailKey> progressCallback) {
         keysDAO.list(url)
-            .flatMap(this::removeAsync, DEFAULT_CONCURRENCY)
+            .flatMap(key -> removeAsync(key).thenReturn(key), DEFAULT_CONCURRENCY)
+            .doOnNext(progressCallback)
             .then()
             .block();
     }
diff --git a/server/mailrepository/mailrepository-memory/src/main/java/org/apache/james/mailrepository/memory/MemoryMailRepository.java b/server/mailrepository/mailrepository-memory/src/main/java/org/apache/james/mailrepository/memory/MemoryMailRepository.java
index 17b53002af..d535e6bdf1 100644
--- a/server/mailrepository/mailrepository-memory/src/main/java/org/apache/james/mailrepository/memory/MemoryMailRepository.java
+++ b/server/mailrepository/mailrepository-memory/src/main/java/org/apache/james/mailrepository/memory/MemoryMailRepository.java
@@ -22,6 +22,7 @@ package org.apache.james.mailrepository.memory;
 import java.util.Iterator;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Consumer;
 
 import jakarta.mail.MessagingException;
 import jakarta.mail.internet.MimeMessage;
@@ -89,6 +90,15 @@ public class MemoryMailRepository implements MailRepository {
         mails.clear();
     }
 
+    @Override
+    public void removeAll(Consumer<MailKey> progressCallback) {
+        mails.keySet()
+            .forEach(key -> {
+                mails.remove(key);
+                progressCallback.accept(key);
+            });
+    }
+
     private Mail cloneMail(Mail mail) {
         try {
             Mail newMail = mail.duplicate();
diff --git a/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/ClearMailRepositoryTask.java b/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/ClearMailRepositoryTask.java
index 877935e91a..5675648df6 100644
--- a/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/ClearMailRepositoryTask.java
+++ b/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/ClearMailRepositoryTask.java
@@ -22,6 +22,7 @@ package org.apache.james.webadmin.service;
 import java.time.Clock;
 import java.time.Instant;
 import java.util.Optional;
+import java.util.concurrent.atomic.AtomicLong;
 
 import jakarta.inject.Inject;
 
@@ -31,7 +32,6 @@ import org.apache.james.mailrepository.api.MailRepositoryStore;
 import org.apache.james.task.Task;
 import org.apache.james.task.TaskExecutionDetails;
 import org.apache.james.task.TaskType;
-import org.reactivestreams.Publisher;
 
 import com.github.fge.lambdas.Throwing;
 
@@ -102,18 +102,24 @@ public class ClearMailRepositoryTask implements Task {
 
     private final MailRepositoryStore mailRepositoryStore;
     private final MailRepositoryPath mailRepositoryPath;
-    private long initialCount = 0;
+    private final AtomicLong initialCount;
+    private final AtomicLong removedCount;
 
     public ClearMailRepositoryTask(MailRepositoryStore mailRepositoryStore, MailRepositoryPath path) {
         this.mailRepositoryStore = mailRepositoryStore;
         this.mailRepositoryPath = path;
+        this.initialCount = new AtomicLong(0);
+        this.removedCount = new AtomicLong(0);
     }
 
     @Override
     public Result run() {
-        initialCount = getRemainingSize().block();
+        initialCount.set(getRemainingSize().block());
         try {
             removeAllInAllRepositories();
+            // Repositories deleting in bulk do not report any progress: only their completion tells us the
+            // repository is now empty.
+            removedCount.set(initialCount.get());
             return Result.COMPLETED;
         } catch (MailRepositoryStore.MailRepositoryStoreException e) {
             LOGGER.error("Encountered error while clearing repository", e);
@@ -123,7 +129,8 @@ public class ClearMailRepositoryTask implements Task {
 
     private void removeAllInAllRepositories() throws MailRepositoryStore.MailRepositoryStoreException {
         mailRepositoryStore.getByPath(mailRepositoryPath)
-            .forEach(Throwing.consumer(MailRepository::removeAll).sneakyThrow());
+            .forEach(Throwing.<MailRepository>consumer(repository -> repository.removeAll(any -> removedCount.incrementAndGet()))
+                .sneakyThrow());
     }
 
     @Override
@@ -135,14 +142,21 @@ public class ClearMailRepositoryTask implements Task {
         return mailRepositoryPath;
     }
 
+    /**
+     * Progress is tracked in memory rather than by counting the remaining mails: counting is a full partition
+     * scan on distributed implementations, and running it upon each and every progress update is prohibitive on
+     * large repositories. Critically, it also used to make the task unable to reach a terminal state whenever
+     * that count failed, as recording completion, failure or cancellation all require these details.
+     */
     @Override
-    public Publisher<Optional<TaskExecutionDetails.AdditionalInformation>> detailsReactive() {
-        return getRemainingSize()
-            .map(remainingSize -> new AdditionalInformation(mailRepositoryPath, initialCount, remainingSize, Clock.systemUTC().instant()))
-            .map(Optional::of);
+    public Optional<TaskExecutionDetails.AdditionalInformation> details() {
+        return Optional.of(new AdditionalInformation(mailRepositoryPath,
+            initialCount.get(),
+            Math.max(initialCount.get() - removedCount.get(), 0),
+            Clock.systemUTC().instant()));
     }
 
-    public Mono<Long> getRemainingSize() {
+    private Mono<Long> getRemainingSize() {
         try {
             return Flux.fromStream(mailRepositoryStore.getByPath(mailRepositoryPath))
                 .flatMap(MailRepository::sizeReactive)
diff --git a/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/service/ClearMailRepositoryTaskTest.java b/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/service/ClearMailRepositoryTaskTest.java
index 51e167a93a..f73b730b3e 100644
--- a/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/service/ClearMailRepositoryTaskTest.java
+++ b/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/service/ClearMailRepositoryTaskTest.java
@@ -19,16 +19,31 @@
 
 package org.apache.james.webadmin.service;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 import java.time.Instant;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
+
+import jakarta.mail.MessagingException;
 
 import org.apache.james.JsonSerializationVerifier;
+import org.apache.james.mailrepository.api.MailKey;
+import org.apache.james.mailrepository.api.MailRepository;
 import org.apache.james.mailrepository.api.MailRepositoryPath;
 import org.apache.james.mailrepository.api.MailRepositoryStore;
 import org.apache.james.server.task.json.JsonTaskSerializer;
 import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentMatchers;
+
+import reactor.core.publisher.Mono;
 
 class ClearMailRepositoryTaskTest {
 
@@ -59,6 +74,88 @@ class ClearMailRepositoryTaskTest {
             .isInstanceOf(ClearMailRepositoryTask.InvalidMailRepositoryPathDeserializationException.class);
     }
 
+    @Test
+    void detailsShouldNotQueryTheRepository() throws Exception {
+        MailRepository repository = mock(MailRepository.class);
+        MailRepositoryStore store = mock(MailRepositoryStore.class);
+        when(store.getByPath(MAIL_REPOSITORY_PATH)).thenAnswer(invocation -> Stream.of(repository));
+
+        ClearMailRepositoryTask task = new ClearMailRepositoryTask(store, MAIL_REPOSITORY_PATH);
+
+        assertThat(task.details()).isPresent();
+        verify(repository, never()).size();
+        verify(repository, never()).sizeReactive();
+    }
+
+    @Test
+    void detailsShouldReportProgressWhileMailsAreBeingRemoved() throws Exception {
+        MailRepository repository = mock(MailRepository.class);
+        when(repository.sizeReactive()).thenReturn(Mono.just(3L));
+        MailRepositoryStore store = mock(MailRepositoryStore.class);
+        when(store.getByPath(MAIL_REPOSITORY_PATH)).thenAnswer(invocation -> Stream.of(repository));
+
+        ClearMailRepositoryTask task = new ClearMailRepositoryTask(store, MAIL_REPOSITORY_PATH);
+        AtomicLong remainingHalfWayThrough = new AtomicLong();
+
+        doAnswer(invocation -> {
+            Consumer<MailKey> progressCallback = invocation.getArgument(0);
+            progressCallback.accept(new MailKey("mail1"));
+            progressCallback.accept(new MailKey("mail2"));
+            remainingHalfWayThrough.set(remainingCount(task));
+            return null;
+        }).when(repository).removeAll(ArgumentMatchers.<Consumer<MailKey>>any());
+
+        task.run();
+
+        assertThat(remainingHalfWayThrough.get()).isEqualTo(1);
+        assertThat(remainingCount(task)).isEqualTo(0);
+    }
+
+    /**
+     * Repositories deleting in bulk report no intermediate progress: only their completion tells us the
+     * repository is now empty.
+     */
+    @Test
+    void detailsShouldReportAnEmptyRepositoryUponCompletionOfABulkRemoval() throws Exception {
+        MailRepository repository = mock(MailRepository.class);
+        when(repository.sizeReactive()).thenReturn(Mono.just(3L));
+        MailRepositoryStore store = mock(MailRepositoryStore.class);
+        when(store.getByPath(MAIL_REPOSITORY_PATH)).thenAnswer(invocation -> Stream.of(repository));
+
+        ClearMailRepositoryTask task = new ClearMailRepositoryTask(store, MAIL_REPOSITORY_PATH);
+
+        task.run();
+
+        assertThat(remainingCount(task)).isEqualTo(0);
+    }
+
+    @Test
+    void detailsShouldNotReportANegativeRemainingCountWhenMailsGetAddedDuringTheRemoval() throws Exception {
+        MailRepository repository = mock(MailRepository.class);
+        when(repository.sizeReactive()).thenReturn(Mono.just(1L));
+        MailRepositoryStore store = mock(MailRepositoryStore.class);
+        when(store.getByPath(MAIL_REPOSITORY_PATH)).thenAnswer(invocation -> Stream.of(repository));
+
+        ClearMailRepositoryTask task = new ClearMailRepositoryTask(store, MAIL_REPOSITORY_PATH);
+        AtomicLong remainingHalfWayThrough = new AtomicLong();
+
+        doAnswer(invocation -> {
+            Consumer<MailKey> progressCallback = invocation.getArgument(0);
+            progressCallback.accept(new MailKey("mail1"));
+            progressCallback.accept(new MailKey("mail2"));
+            remainingHalfWayThrough.set(remainingCount(task));
+            return null;
+        }).when(repository).removeAll(ArgumentMatchers.<Consumer<MailKey>>any());
+
+        task.run();
+
+        assertThat(remainingHalfWayThrough.get()).isEqualTo(0);
+    }
+
+    private long remainingCount(ClearMailRepositoryTask task) throws MessagingException {
+        return ((ClearMailRepositoryTask.AdditionalInformation) task.details().get()).getRemainingCount();
+    }
+
     @Test
     void additionalInformationShouldBeSerializable() throws Exception {
         JsonSerializationVerifier.dtoModule(ClearMailRepositoryTaskAdditionalInformationDTO.module())


---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org


(james-project) 06/07: [FIX] Toletate failures for Task additional information

Posted by hq...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

quantranhong1999 pushed a commit to branch 3.9.x
in repository https://gitbox.apache.org/repos/asf/james-project.git

commit d3f6ef0b0869bb3106ba11d87e5635fc044f8856
Author: Benoit TELLIER <bt...@linagora.com>
AuthorDate: Sun Aug 16 23:38:17 2026 +0700

    [FIX] Toletate failures for Task additional information
---
 .../distributed/RabbitMQWorkQueue.java             |  3 ++
 .../apache/james/task/SerialTaskManagerWorker.java | 20 +++++++-
 .../task/eventsourcing/WorkerStatusListener.scala  | 34 ++++++++++---
 .../EventSourcingTaskManagerTest.java              | 59 ++++++++++++++++++++++
 4 files changed, 108 insertions(+), 8 deletions(-)

diff --git a/server/task/task-distributed/src/main/java/org/apache/james/task/eventsourcing/distributed/RabbitMQWorkQueue.java b/server/task/task-distributed/src/main/java/org/apache/james/task/eventsourcing/distributed/RabbitMQWorkQueue.java
index 48179512a4..94500011bf 100644
--- a/server/task/task-distributed/src/main/java/org/apache/james/task/eventsourcing/distributed/RabbitMQWorkQueue.java
+++ b/server/task/task-distributed/src/main/java/org/apache/james/task/eventsourcing/distributed/RabbitMQWorkQueue.java
@@ -164,6 +164,9 @@ public class RabbitMQWorkQueue implements WorkQueue {
             .map(taskIdValue -> TaskId.fromString(taskIdValue.toString()))
             .flatMap(taskId -> Mono.fromCallable(() -> new String(delivery.getBody(), StandardCharsets.UTF_8))
                 .flatMap(bodyValue -> deserialize(bodyValue, taskId))
+                // Deserialization failures yield no task: the delivery would otherwise be neither acked nor
+                // nacked, and thus be redelivered over and over.
+                .switchIfEmpty(Mono.fromRunnable(() -> delivery.nack(!REQUEUE)))
                 .doOnNext(task -> delivery.ack())
                 .flatMap(task -> executeOnWorker(taskId, task))
                 .doOnSuccess(result -> LOGGER.info("Executed task {} yield {}", taskId, result)))
diff --git a/server/task/task-memory/src/main/java/org/apache/james/task/SerialTaskManagerWorker.java b/server/task/task-memory/src/main/java/org/apache/james/task/SerialTaskManagerWorker.java
index 6144bd7e88..fee70fcfa2 100644
--- a/server/task/task-memory/src/main/java/org/apache/james/task/SerialTaskManagerWorker.java
+++ b/server/task/task-memory/src/main/java/org/apache/james/task/SerialTaskManagerWorker.java
@@ -81,7 +81,9 @@ public class SerialTaskManagerWorker implements TaskManagerWorker {
             runningTasks.put(taskWithId.getId(), future);
 
             Mono<Task.Result> pollingMono = Mono.using(
-                () -> pollAdditionalInformation(taskWithId).subscribe(),
+                () -> pollAdditionalInformation(taskWithId).subscribe(
+                    any -> { },
+                    e -> LOGGER.error("Stopped polling additional information updates of task {}", taskWithId.getId().asString(), e)),
                 ignored -> Mono.fromFuture(future)
                     .onErrorResume(exception -> Mono.from(handleExecutionError(taskWithId, listener, exception))
                         .thenReturn(Task.Result.PARTIAL)),
@@ -119,9 +121,23 @@ public class SerialTaskManagerWorker implements TaskManagerWorker {
         }
     }
 
+    /**
+     * Polls the progress of a task until it terminates.
+     *
+     * Computing the additional information of a task can fail - typically when it is backed by the very data
+     * the task is struggling with. Such a failure needs to be swallowed: it would otherwise terminate this
+     * Flux, silently depriving the task of any progress reporting for the rest of its execution.
+     *
+     * Note that the delay needs to be applied upon subscription rather than upon emission: an errored, hence
+     * empty, poll would otherwise complete right away and have `repeat` resubscribe in a tight loop.
+     */
     private Flux<TaskExecutionDetails.AdditionalInformation> pollAdditionalInformation(TaskWithId taskWithId) {
         return Mono.from(taskWithId.getTask().detailsReactive())
-            .delayElement(pollingInterval, Schedulers.parallel())
+            .onErrorResume(e -> {
+                LOGGER.error("Error while computing additional information updates of task {}", taskWithId.getId().asString(), e);
+                return Mono.empty();
+            })
+            .delaySubscription(pollingInterval, Schedulers.parallel())
             .repeat()
             .handle(publishIfPresent())
             .flatMap(information -> Mono.from(listener.updated(taskWithId.getId(), Mono.just(information)))
diff --git a/server/task/task-memory/src/main/scala/org/apache/james/task/eventsourcing/WorkerStatusListener.scala b/server/task/task-memory/src/main/scala/org/apache/james/task/eventsourcing/WorkerStatusListener.scala
index 642e6fa1a4..b4e2bc24a4 100644
--- a/server/task/task-memory/src/main/scala/org/apache/james/task/eventsourcing/WorkerStatusListener.scala
+++ b/server/task/task-memory/src/main/scala/org/apache/james/task/eventsourcing/WorkerStatusListener.scala
@@ -25,30 +25,52 @@ import org.apache.james.task.Task.Result
 import org.apache.james.task.eventsourcing.TaskCommand._
 import org.apache.james.task.{TaskExecutionDetails, TaskId, TaskManagerWorker}
 import org.reactivestreams.Publisher
+import org.slf4j.{Logger, LoggerFactory}
 import reactor.core.scala.publisher.SMono
 
 import java.util.Optional
 import scala.jdk.OptionConverters._
 
+object WorkerStatusListener {
+  private val LOGGER: Logger = LoggerFactory.getLogger(classOf[WorkerStatusListener])
+}
+
 case class WorkerStatusListener(eventSourcingSystem: EventSourcingSystem) extends TaskManagerWorker.Listener {
 
+  import WorkerStatusListener.LOGGER
+
   override def started(taskId: TaskId): Publisher[Void] = SMono(eventSourcingSystem.dispatch(Start(taskId))).`then`()
 
   override def completed(taskId: TaskId, result: Result, additionalInformationPublisher: Publisher[Optional[TaskExecutionDetails.AdditionalInformation]]): Publisher[Void] =
-    SMono.fromPublisher(additionalInformationPublisher)
-      .flatMap(additionalInformation => SMono(eventSourcingSystem.dispatch(Complete(taskId, result, additionalInformation.toScala))))
+    additionalInformation(taskId, additionalInformationPublisher)
+      .flatMap(additionalInformation => SMono(eventSourcingSystem.dispatch(Complete(taskId, result, additionalInformation))))
       .`then`()
 
   override def failed(taskId: TaskId, additionalInformationPublisher: Publisher[Optional[TaskExecutionDetails.AdditionalInformation]], errorMessage: Optional[String], t: Optional[Throwable]): Publisher[Void] =
-    SMono.fromPublisher(additionalInformationPublisher)
-    .flatMap(additionalInformation => SMono(eventSourcingSystem.dispatch(Fail(taskId, additionalInformation.toScala, errorMessage.toScala, t.toScala.map(t => Throwables.getStackTraceAsString(t))))))
+    additionalInformation(taskId, additionalInformationPublisher)
+    .flatMap(additionalInformation => SMono(eventSourcingSystem.dispatch(Fail(taskId, additionalInformation, errorMessage.toScala, t.toScala.map(t => Throwables.getStackTraceAsString(t))))))
     .`then`()
 
   override def cancelled(taskId: TaskId, additionalInformationPublisher: Publisher[Optional[TaskExecutionDetails.AdditionalInformation]]): Publisher[Void] =
-    SMono.fromPublisher(additionalInformationPublisher)
-      .flatMap(additionalInformation => SMono(eventSourcingSystem.dispatch(Cancel(taskId, additionalInformation.toScala))))
+    additionalInformation(taskId, additionalInformationPublisher)
+      .flatMap(additionalInformation => SMono(eventSourcingSystem.dispatch(Cancel(taskId, additionalInformation))))
       .`then`()
 
+  /**
+   * A task needs to be able to reach a terminal state even when its progress reporting is broken: some tasks
+   * compute their additional information by querying the very data they are working on, which can well be the
+   * failure being recorded. Letting such an error propagate would skip the dispatch altogether, leaving the
+   * task in progress forever - neither completable, nor failable, nor cancellable.
+   */
+  private def additionalInformation(taskId: TaskId, additionalInformationPublisher: Publisher[Optional[TaskExecutionDetails.AdditionalInformation]]): SMono[Option[TaskExecutionDetails.AdditionalInformation]] =
+    SMono.fromPublisher(additionalInformationPublisher)
+      .map(_.toScala)
+      .onErrorResume(e => {
+        LOGGER.warn("Could not retrieve additional information of task {}. Recording its outcome without it.", taskId.asString(), e)
+        SMono.just(None)
+      })
+      .defaultIfEmpty(None)
+
   override def updated(taskId: TaskId, additionalInformationPublisher: Publisher[TaskExecutionDetails.AdditionalInformation]): Publisher[Void] =
     SMono.fromPublisher(additionalInformationPublisher)
       .flatMap(additionalInformation => SMono(eventSourcingSystem.dispatch(UpdateAdditionalInformation(taskId, additionalInformation))))
diff --git a/server/task/task-memory/src/test/java/org/apache/james/task/eventsourcing/EventSourcingTaskManagerTest.java b/server/task/task-memory/src/test/java/org/apache/james/task/eventsourcing/EventSourcingTaskManagerTest.java
index f54f61d042..8956e93b22 100644
--- a/server/task/task-memory/src/test/java/org/apache/james/task/eventsourcing/EventSourcingTaskManagerTest.java
+++ b/server/task/task-memory/src/test/java/org/apache/james/task/eventsourcing/EventSourcingTaskManagerTest.java
@@ -25,6 +25,7 @@ import static org.assertj.core.api.Assertions.assertThatCode;
 import java.time.ZonedDateTime;
 import java.time.temporal.ChronoUnit;
 import java.util.Optional;
+import java.util.concurrent.CountDownLatch;
 
 import org.apache.james.eventsourcing.eventstore.EventStore;
 import org.apache.james.eventsourcing.eventstore.memory.InMemoryEventStore;
@@ -44,6 +45,8 @@ import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.function.ThrowingSupplier;
+import org.reactivestreams.Publisher;
 
 import reactor.core.publisher.Mono;
 
@@ -114,6 +117,62 @@ class EventSourcingTaskManagerTest implements TaskManagerContract {
                 .containsOnly(HOSTNAME));
     }
 
+    @Test
+    void taskShouldCompleteWhenAdditionalInformationCanNotBeComputed() {
+        TaskId taskId = taskManager.submit(new BrokenDetailsTask(() -> Task.Result.COMPLETED));
+
+        awaitUntilTaskHasStatus(taskId, TaskManager.Status.COMPLETED, taskManager);
+    }
+
+    @Test
+    void taskShouldFailWhenAdditionalInformationCanNotBeComputed() {
+        TaskId taskId = taskManager.submit(new BrokenDetailsTask(() -> Task.Result.PARTIAL));
+
+        awaitUntilTaskHasStatus(taskId, TaskManager.Status.FAILED, taskManager);
+    }
+
+    @Test
+    void taskShouldBeCancelledWhenAdditionalInformationCanNotBeComputed(CountDownLatch countDownLatch) {
+        TaskId taskId = taskManager.submit(new BrokenDetailsTask(() -> {
+            countDownLatch.await();
+            return Task.Result.COMPLETED;
+        }));
+
+        awaitUntilTaskHasStatus(taskId, TaskManager.Status.IN_PROGRESS, taskManager);
+        taskManager.cancel(taskId);
+        countDownLatch.countDown();
+
+        awaitUntilTaskHasStatus(taskId, TaskManager.Status.CANCELLED, taskManager);
+    }
+
+    /**
+     * A task computing its additional information out of the very data it is working on: its progress reporting
+     * breaks down exactly when things go wrong. Recording the outcome of such a task needs to keep working, as
+     * it would otherwise be left in progress forever.
+     */
+    private static class BrokenDetailsTask implements Task {
+        private final ThrowingSupplier<Result> task;
+
+        BrokenDetailsTask(ThrowingSupplier<Result> task) {
+            this.task = task;
+        }
+
+        @Override
+        public Result run() throws InterruptedException {
+            return new MemoryReferenceTask(task).run();
+        }
+
+        @Override
+        public TaskType type() {
+            return TaskType.of("broken-details");
+        }
+
+        @Override
+        public Publisher<Optional<TaskExecutionDetails.AdditionalInformation>> detailsReactive() {
+            return Mono.error(new RuntimeException("Additional information can not be computed"));
+        }
+    }
+
     @Test
     void cancelShouldNotFailWhenExecutionDetailsHaveNoEvents() {
         TaskId taskId = TaskId.generateTaskId();


---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org


(james-project) 01/07: [FIX] Handle missing Task history in Cassandra implem like PG

Posted by hq...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

quantranhong1999 pushed a commit to branch 3.9.x
in repository https://gitbox.apache.org/repos/asf/james-project.git

commit 67701826a7f19d5dbff6c07b848787d4a7a09f2c
Author: Benoit TELLIER <bt...@linagora.com>
AuthorDate: Sun Aug 16 16:22:29 2026 +0700

    [FIX] Handle missing Task history in Cassandra implem like PG
---
 .../eventstore/cassandra/CassandraEventStore.scala    |  1 +
 .../cassandra/CassandraEventStoreExtension.scala      | 12 +++++++++---
 .../cassandra/CassandraEventStoreTest.scala           | 19 +++++++++++++++++++
 3 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/event-sourcing/event-store-cassandra/src/main/scala/org/apache/james/eventsourcing/eventstore/cassandra/CassandraEventStore.scala b/event-sourcing/event-store-cassandra/src/main/scala/org/apache/james/eventsourcing/eventstore/cassandra/CassandraEventStore.scala
index 9266a7e31a..2eef0a9816 100644
--- a/event-sourcing/event-store-cassandra/src/main/scala/org/apache/james/eventsourcing/eventstore/cassandra/CassandraEventStore.scala
+++ b/event-sourcing/event-store-cassandra/src/main/scala/org/apache/james/eventsourcing/eventstore/cassandra/CassandraEventStore.scala
@@ -49,6 +49,7 @@ class CassandraEventStore @Inject() (eventStoreDao: EventStoreDao) extends Event
   override def getEventsOfAggregate(aggregateId: AggregateId): SMono[History] =
     eventStoreDao.getSnapshot(aggregateId)
       .flatMap(snapshotId => eventStoreDao.getEventsOfAggregate(aggregateId, snapshotId))
+      .filter(history => history.getEvents.nonEmpty)
       .switchIfEmpty(eventStoreDao.getEventsOfAggregate(aggregateId))
 
   override def remove(aggregateId: AggregateId): Publisher[Void] = eventStoreDao.delete(aggregateId)
diff --git a/event-sourcing/event-store-cassandra/src/test/scala/org/apache/james/eventsourcing/eventstore/cassandra/CassandraEventStoreExtension.scala b/event-sourcing/event-store-cassandra/src/test/scala/org/apache/james/eventsourcing/eventstore/cassandra/CassandraEventStoreExtension.scala
index d16dae4ee2..1662f22fab 100644
--- a/event-sourcing/event-store-cassandra/src/test/scala/org/apache/james/eventsourcing/eventstore/cassandra/CassandraEventStoreExtension.scala
+++ b/event-sourcing/event-store-cassandra/src/test/scala/org/apache/james/eventsourcing/eventstore/cassandra/CassandraEventStoreExtension.scala
@@ -18,6 +18,7 @@
  ****************************************************************/
 package org.apache.james.eventsourcing.eventstore.cassandra
 
+import com.datastax.oss.driver.api.core.CqlSession
 import org.apache.james.backends.cassandra.CassandraClusterExtension
 import org.apache.james.eventsourcing.eventstore.{EventStore, JsonEventSerializer}
 import org.junit.jupiter.api.extension.{AfterAllCallback, AfterEachCallback, BeforeAllCallback, BeforeEachCallback, ExtensionContext, ParameterContext, ParameterResolutionException, ParameterResolver}
@@ -40,9 +41,14 @@ class CassandraEventStoreExtension(var cassandra: CassandraClusterExtension, val
 
   @throws[ParameterResolutionException]
   override def supportsParameter(parameterContext: ParameterContext, extensionContext: ExtensionContext): Boolean =
-    parameterContext.getParameter.getType eq classOf[EventStore]
+    (parameterContext.getParameter.getType eq classOf[EventStore]) ||
+      (parameterContext.getParameter.getType eq classOf[CqlSession])
 
   @throws[ParameterResolutionException]
-  override def resolveParameter(parameterContext: ParameterContext, extensionContext: ExtensionContext): CassandraEventStore =
-    new CassandraEventStore(eventStoreDao.get)
+  override def resolveParameter(parameterContext: ParameterContext, extensionContext: ExtensionContext): AnyRef =
+    if (parameterContext.getParameter.getType eq classOf[CqlSession]) {
+      cassandra.getCassandraCluster.getConf
+    } else {
+      new CassandraEventStore(eventStoreDao.get)
+    }
 }
\ No newline at end of file
diff --git a/event-sourcing/event-store-cassandra/src/test/scala/org/apache/james/eventsourcing/eventstore/cassandra/CassandraEventStoreTest.scala b/event-sourcing/event-store-cassandra/src/test/scala/org/apache/james/eventsourcing/eventstore/cassandra/CassandraEventStoreTest.scala
index 718f37c842..c55b42f824 100644
--- a/event-sourcing/event-store-cassandra/src/test/scala/org/apache/james/eventsourcing/eventstore/cassandra/CassandraEventStoreTest.scala
+++ b/event-sourcing/event-store-cassandra/src/test/scala/org/apache/james/eventsourcing/eventstore/cassandra/CassandraEventStoreTest.scala
@@ -18,6 +18,8 @@
  ****************************************************************/
 package org.apache.james.eventsourcing.eventstore.cassandra
 
+import com.datastax.oss.driver.api.core.CqlSession
+import com.datastax.oss.driver.api.querybuilder.QueryBuilder.{literal, update}
 import org.apache.james.eventsourcing.eventstore.dto.SnapshotEvent
 import org.apache.james.eventsourcing.{EventId, TestEvent}
 import org.apache.james.eventsourcing.eventstore.{EventStore, EventStoreContract, History}
@@ -55,4 +57,21 @@ class CassandraEventStoreTest extends EventStoreContract {
     assertThat(SMono(testee.getEventsOfAggregate(EventStoreContract.AGGREGATE_1)).block())
       .isEqualTo(History.of(event3))
   }
+
+  @Test
+  def getEventsOfAggregateShouldFallBackToFullHistoryWhenSnapshotPointsToMissingEvents(testee: EventStore, session: CqlSession) : Unit = {
+    val event1 = TestEvent(EventId.first, EventStoreContract.AGGREGATE_1, "first")
+
+    SMono(testee.append(event1)).block()
+
+    // The snapshot is a static column written by a blind update, outside of the LWT appending the events:
+    // it can thus end up referencing events that are no longer part of the partition.
+    session.execute(update(CassandraEventStoreTable.EVENTS_TABLE)
+      .setColumn(CassandraEventStoreTable.SNAPSHOT, literal(5))
+      .whereColumn(CassandraEventStoreTable.AGGREGATE_ID).isEqualTo(literal(EventStoreContract.AGGREGATE_1.asAggregateKey))
+      .build())
+
+    assertThat(SMono(testee.getEventsOfAggregate(EventStoreContract.AGGREGATE_1)).block())
+      .isEqualTo(History.of(event1))
+  }
 }
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org


(james-project) 03/07: [FIX] Task cleanup should purge stalled tasks

Posted by hq...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

quantranhong1999 pushed a commit to branch 3.9.x
in repository https://gitbox.apache.org/repos/asf/james-project.git

commit 7762f6085efd77e5df9f981319bf4bdf6db725fd
Author: Benoit TELLIER <bt...@linagora.com>
AuthorDate: Sun Aug 16 16:33:29 2026 +0700

    [FIX] Task cleanup should purge stalled tasks
---
 .../webadmin/services/TasksCleanupService.java     | 47 +++++++++++-
 .../webadmin/routes/TasksCleanupRoutesTest.java    | 86 +++++++++++++++++++++-
 2 files changed, 129 insertions(+), 4 deletions(-)

diff --git a/server/protocols/webadmin/webadmin-cassandra/src/main/java/org/apache/james/webadmin/services/TasksCleanupService.java b/server/protocols/webadmin/webadmin-cassandra/src/main/java/org/apache/james/webadmin/services/TasksCleanupService.java
index 6609083407..677908210c 100644
--- a/server/protocols/webadmin/webadmin-cassandra/src/main/java/org/apache/james/webadmin/services/TasksCleanupService.java
+++ b/server/protocols/webadmin/webadmin-cassandra/src/main/java/org/apache/james/webadmin/services/TasksCleanupService.java
@@ -20,6 +20,7 @@
 package org.apache.james.webadmin.services;
 
 import java.time.Instant;
+import java.time.ZonedDateTime;
 import java.util.concurrent.atomic.AtomicLong;
 
 import jakarta.inject.Inject;
@@ -27,6 +28,7 @@ import jakarta.inject.Inject;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.james.eventsourcing.eventstore.EventStore;
 import org.apache.james.task.Task;
+import org.apache.james.task.TaskExecutionDetails;
 import org.apache.james.task.TaskId;
 import org.apache.james.task.TaskManager;
 import org.apache.james.task.eventsourcing.TaskAggregateId;
@@ -108,8 +110,7 @@ public class TasksCleanupService {
 
     private Flux<Pair<TaskId, Task.Result>> removeTask(Instant beforeDate) {
         return Flux.from(taskExecutionDetailsProjection.listDetailsByBeforeDate(beforeDate))
-            .filter(oldTaskDetail -> !(oldTaskDetail.getStatus().equals(TaskManager.Status.WAITING)
-                || oldTaskDetail.getStatus().equals(TaskManager.Status.IN_PROGRESS)))
+            .filterWhen(oldTaskDetail -> canBeRemoved(oldTaskDetail, beforeDate))
             .flatMap(oldTaskDetail -> Mono.from(eventStore.remove(new TaskAggregateId(oldTaskDetail.getTaskId())))
                 .then(Mono.from(taskExecutionDetailsProjection.remove(oldTaskDetail)))
                 .then(Mono.just(Pair.of(oldTaskDetail.getTaskId(), Task.Result.COMPLETED)))
@@ -119,6 +120,48 @@ public class TasksCleanupService {
                 }));
     }
 
+    private Mono<Boolean> canBeRemoved(TaskExecutionDetails taskDetail, Instant beforeDate) {
+        if (!isUnfinished(taskDetail)) {
+            return Mono.just(true);
+        }
+        if (isStale(taskDetail, beforeDate)) {
+            LOGGER.warn("Removing task {} of type {}, left in {} status with no activity since {}: " +
+                    "the node running it died before completing it.",
+                taskDetail.getTaskId().asString(), taskDetail.getType().asString(),
+                taskDetail.getStatus().getValue(), lastActivity(taskDetail));
+            return Mono.just(true);
+        }
+        // Still showing signs of life, yet a task without any event can not be running either: its history
+        // was lost. As cancelling it has no effect, removing it is the only way to get rid of such an entry.
+        return Mono.from(eventStore.getEventsOfAggregate(new TaskAggregateId(taskDetail.getTaskId())))
+            .map(history -> history.getEventsJava().isEmpty());
+    }
+
+    private boolean isUnfinished(TaskExecutionDetails taskDetail) {
+        return taskDetail.getStatus().equals(TaskManager.Status.WAITING)
+            || taskDetail.getStatus().equals(TaskManager.Status.IN_PROGRESS);
+    }
+
+    /**
+     * A task being executed keeps refreshing its execution details. One left untouched since the cleanup
+     * horizon lost the node running it, and would otherwise stay in the listing forever: such an entry can
+     * neither be cancelled nor deleted through any other route.
+     */
+    private boolean isStale(TaskExecutionDetails taskDetail, Instant beforeDate) {
+        return lastActivity(taskDetail).isBefore(beforeDate);
+    }
+
+    private Instant lastActivity(TaskExecutionDetails taskDetail) {
+        Instant lastKnownDate = taskDetail.getStartedDate()
+            .map(ZonedDateTime::toInstant)
+            .orElseGet(() -> taskDetail.getSubmittedDate().toInstant());
+
+        return taskDetail.getAdditionalInformation()
+            .map(TaskExecutionDetails.AdditionalInformation::timestamp)
+            .filter(lastKnownDate::isBefore)
+            .orElse(lastKnownDate);
+    }
+
     private static void doOnNext(Pair<TaskId, Task.Result> next, Context context) {
         context.incrementProcessedTaskCount();
         if (Task.Result.COMPLETED.equals(next.getValue())) {
diff --git a/server/protocols/webadmin/webadmin-cassandra/src/test/java/org/apache/james/webadmin/routes/TasksCleanupRoutesTest.java b/server/protocols/webadmin/webadmin-cassandra/src/test/java/org/apache/james/webadmin/routes/TasksCleanupRoutesTest.java
index cf91f44c08..fd9887210a 100644
--- a/server/protocols/webadmin/webadmin-cassandra/src/test/java/org/apache/james/webadmin/routes/TasksCleanupRoutesTest.java
+++ b/server/protocols/webadmin/webadmin-cassandra/src/test/java/org/apache/james/webadmin/routes/TasksCleanupRoutesTest.java
@@ -319,11 +319,12 @@ public class TasksCleanupRoutesTest {
 
     @ParameterizedTest
     @MethodSource(value = "inProgressStatus")
-    void tasksCleanupShouldNotRemoveInProgressTask(TaskManager.Status status) {
+    void tasksCleanupShouldRemoveStaleUnfinishedTask(TaskManager.Status status) {
+        // Not updated since the cleanup horizon: the node running it is gone.
         TaskExecutionDetails taskExecutionDetail = new TaskExecutionDetails(TaskId.generateTaskId(),
             TaskType.of("type"),
             status,
-            ZonedDateTime.now(),
+            ZonedDateTime.now().minus(20, ChronoUnit.DAYS),
             new Hostname("foo"),
             Optional::empty,
             Optional.empty(),
@@ -333,6 +334,87 @@ public class TasksCleanupRoutesTest {
             Optional.empty(),
             Optional.empty());
 
+        taskExecutionDetailsProjection.update(taskExecutionDetail);
+        Created event = new Created(new TaskAggregateId(taskExecutionDetail.taskId()), EventId.first(), new MemoryReferenceWithCounterTask((counter) -> Task.Result.COMPLETED), new Hostname("foo"));
+        Mono.from(eventStore.append(event)).block();
+
+        String taskId = given()
+            .queryParam("olderThan", "15day")
+            .delete()
+            .jsonPath()
+            .get("taskId");
+
+        given()
+            .basePath(TasksRoutes.BASE)
+        .when()
+            .get(taskId + "/await")
+        .then()
+            .body("status", is("completed"))
+            .body("taskId", is(taskId))
+            .body("type", is("tasks-cleanup"))
+            .body("additionalInformation.removedTaskCount", is(1))
+            .body("additionalInformation.processedTaskCount", is(1));
+
+        assertThat(taskExecutionDetailsProjection.list().size())
+            .isEqualTo(0);
+    }
+
+    @Test
+    void tasksCleanupShouldRemoveUnfinishedTaskWhenEventsAreMissing() {
+        // Still recently active, yet the event store holds no history for it: the entry can not be
+        // running, and neither cancelling nor deleting it individually gets rid of it.
+        TaskExecutionDetails taskExecutionDetail = new TaskExecutionDetails(TaskId.generateTaskId(),
+            TaskType.of("type"),
+            TaskManager.Status.IN_PROGRESS,
+            ZonedDateTime.now().minus(20, ChronoUnit.DAYS),
+            new Hostname("foo"),
+            Optional::empty,
+            Optional.of(ZonedDateTime.now()),
+            Optional.empty(),
+            Optional.empty(),
+            Optional.empty(),
+            Optional.empty(),
+            Optional.empty());
+
+        taskExecutionDetailsProjection.update(taskExecutionDetail);
+
+        String taskId = given()
+            .queryParam("olderThan", "15day")
+            .delete()
+            .jsonPath()
+            .get("taskId");
+
+        given()
+            .basePath(TasksRoutes.BASE)
+        .when()
+            .get(taskId + "/await")
+        .then()
+            .body("status", is("completed"))
+            .body("taskId", is(taskId))
+            .body("type", is("tasks-cleanup"))
+            .body("additionalInformation.removedTaskCount", is(1))
+            .body("additionalInformation.processedTaskCount", is(1));
+
+        assertThat(taskExecutionDetailsProjection.list().size())
+            .isEqualTo(0);
+    }
+
+    @Test
+    void tasksCleanupShouldNotRemoveInProgressTask() {
+        // Started recently: the task is genuinely being executed, whatever the age of its submission.
+        TaskExecutionDetails taskExecutionDetail = new TaskExecutionDetails(TaskId.generateTaskId(),
+            TaskType.of("type"),
+            TaskManager.Status.IN_PROGRESS,
+            ZonedDateTime.now().minus(20, ChronoUnit.DAYS),
+            new Hostname("foo"),
+            Optional::empty,
+            Optional.of(ZonedDateTime.now()),
+            Optional.empty(),
+            Optional.empty(),
+            Optional.empty(),
+            Optional.empty(),
+            Optional.empty());
+
         taskExecutionDetailsProjection.update(taskExecutionDetail);
         TaskAggregateId taskAggregateId = new TaskAggregateId(taskExecutionDetail.taskId());
         Created event = new Created(taskAggregateId, EventId.first(), new MemoryReferenceWithCounterTask((counter) -> Task.Result.COMPLETED), new Hostname("foo"));


---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org


(james-project) 05/07: [FIX] Single row to read eventStore snapshots

Posted by hq...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

quantranhong1999 pushed a commit to branch 3.9.x
in repository https://gitbox.apache.org/repos/asf/james-project.git

commit ec435cd5218293b796bad44bdec82edf632db273
Author: Benoit TELLIER <bt...@linagora.com>
AuthorDate: Sun Aug 16 16:44:26 2026 +0700

    [FIX] Single row to read eventStore snapshots
---
 .../apache/james/eventsourcing/eventstore/cassandra/EventStoreDao.scala  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/event-sourcing/event-store-cassandra/src/main/scala/org/apache/james/eventsourcing/eventstore/cassandra/EventStoreDao.scala b/event-sourcing/event-store-cassandra/src/main/scala/org/apache/james/eventsourcing/eventstore/cassandra/EventStoreDao.scala
index 63f013d925..75b74e2288 100644
--- a/event-sourcing/event-store-cassandra/src/main/scala/org/apache/james/eventsourcing/eventstore/cassandra/EventStoreDao.scala
+++ b/event-sourcing/event-store-cassandra/src/main/scala/org/apache/james/eventsourcing/eventstore/cassandra/EventStoreDao.scala
@@ -73,6 +73,7 @@ class EventStoreDao @Inject() (val session: CqlSession,
       .selectFrom(EVENTS_TABLE)
       .column(SNAPSHOT)
       .whereColumn(AGGREGATE_ID).isEqualTo(bindMarker(AGGREGATE_ID))
+      .limit(1)
       .build())
 
   private def prepareSelectFrom(session: CqlSession): PreparedStatement =


---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org


(james-project) 04/07: [FIX] Allow canceling tasks with truncated history

Posted by hq...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

quantranhong1999 pushed a commit to branch 3.9.x
in repository https://gitbox.apache.org/repos/asf/james-project.git

commit b81c2473653c9805902649256e5ce422eed41a07
Author: Benoit TELLIER <bt...@linagora.com>
AuthorDate: Sun Aug 16 16:38:40 2026 +0700

    [FIX] Allow canceling tasks with truncated history
---
 .../james/task/eventsourcing/CommandHandlers.scala | 10 ++++--
 .../EventSourcingTaskManagerTest.java              | 39 +++++++++++++++++++++-
 2 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/server/task/task-memory/src/main/scala/org/apache/james/task/eventsourcing/CommandHandlers.scala b/server/task/task-memory/src/main/scala/org/apache/james/task/eventsourcing/CommandHandlers.scala
index e1c4ab8309..549c618b76 100644
--- a/server/task/task-memory/src/main/scala/org/apache/james/task/eventsourcing/CommandHandlers.scala
+++ b/server/task/task-memory/src/main/scala/org/apache/james/task/eventsourcing/CommandHandlers.scala
@@ -59,7 +59,13 @@ class RequestCancelCommandHandler(private val loadHistory: TaskAggregateId => SM
   override def handledClass: Class[RequestCancel] = classOf[RequestCancel]
 
   override def handle(command: RequestCancel): Publisher[List[EventWithState]] = {
-    loadAggregate(loadHistory, command.id).map(_.requestCancel(hostname).map(EventWithState.noState).asJava)
+    val aggregateId = TaskAggregateId(command.id)
+    loadHistory(aggregateId).map(history =>
+      if (history.getEvents.isEmpty) {
+        Seq.empty[EventWithState].asJava
+      } else {
+        TaskAggregate.fromHistory(aggregateId, history).requestCancel(hostname).map(EventWithState.noState).asJava
+      })
   }
 }
 
@@ -93,4 +99,4 @@ class UpdateCommandHandler(private val loadHistory: TaskAggregateId => SMono[His
   override def handle(command: UpdateAdditionalInformation): Publisher[List[EventWithState]] = {
     loadAggregate(loadHistory, command.id).map(_.update(command.additionalInformation).map(EventWithState.noState).asJava)
   }
-}
\ No newline at end of file
+}
diff --git a/server/task/task-memory/src/test/java/org/apache/james/task/eventsourcing/EventSourcingTaskManagerTest.java b/server/task/task-memory/src/test/java/org/apache/james/task/eventsourcing/EventSourcingTaskManagerTest.java
index fb6f30ff1d..f54f61d042 100644
--- a/server/task/task-memory/src/test/java/org/apache/james/task/eventsourcing/EventSourcingTaskManagerTest.java
+++ b/server/task/task-memory/src/test/java/org/apache/james/task/eventsourcing/EventSourcingTaskManagerTest.java
@@ -20,6 +20,11 @@
 package org.apache.james.task.eventsourcing;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatCode;
+
+import java.time.ZonedDateTime;
+import java.time.temporal.ChronoUnit;
+import java.util.Optional;
 
 import org.apache.james.eventsourcing.eventstore.EventStore;
 import org.apache.james.eventsourcing.eventstore.memory.InMemoryEventStore;
@@ -29,10 +34,12 @@ import org.apache.james.task.MemoryReferenceTask;
 import org.apache.james.task.MemoryWorkQueue;
 import org.apache.james.task.SerialTaskManagerWorker;
 import org.apache.james.task.Task;
+import org.apache.james.task.TaskExecutionDetails;
 import org.apache.james.task.TaskId;
 import org.apache.james.task.TaskManager;
 import org.apache.james.task.TaskManagerContract;
 import org.apache.james.task.TaskManagerWorker;
+import org.apache.james.task.TaskType;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -45,11 +52,12 @@ class EventSourcingTaskManagerTest implements TaskManagerContract {
     private static final Hostname HOSTNAME = new Hostname("foo");
     private EventSourcingTaskManager taskManager;
     private EventStore eventStore;
+    private TaskExecutionDetailsProjection executionDetailsProjection;
 
     @BeforeEach
     void setUp() {
         eventStore = new InMemoryEventStore();
-        TaskExecutionDetailsProjection executionDetailsProjection = new MemoryTaskExecutionDetailsProjection();
+        executionDetailsProjection = new MemoryTaskExecutionDetailsProjection();
         WorkQueueSupplier workQueueSupplier = eventSourcingSystem -> {
             WorkerStatusListener listener = new WorkerStatusListener(eventSourcingSystem);
             TaskManagerWorker worker = new SerialTaskManagerWorker(listener, UPDATE_INFORMATION_POLLING_INTERVAL);
@@ -105,4 +113,33 @@ class EventSourcingTaskManagerTest implements TaskManagerContract {
                 .extracting("hostname")
                 .containsOnly(HOSTNAME));
     }
+
+    @Test
+    void cancelShouldNotFailWhenExecutionDetailsHaveNoEvents() {
+        TaskId taskId = TaskId.generateTaskId();
+        executionDetailsProjection.update(unfinishedDetailsWithoutEvents(taskId));
+
+        assertThatCode(() -> taskManager.cancel(taskId))
+            .doesNotThrowAnyException();
+    }
+
+    /**
+     * An entry of the execution details projection whose events are missing from the event store: such an
+     * inconsistency is caused by a data loss on the event store, and used to make the task impossible to
+     * interact with at all.
+     */
+    private TaskExecutionDetails unfinishedDetailsWithoutEvents(TaskId taskId) {
+        return new TaskExecutionDetails(taskId,
+            TaskType.of("type"),
+            TaskManager.Status.IN_PROGRESS,
+            ZonedDateTime.now().minus(20, ChronoUnit.DAYS),
+            HOSTNAME,
+            Optional::empty,
+            Optional.empty(),
+            Optional.empty(),
+            Optional.empty(),
+            Optional.empty(),
+            Optional.empty(),
+            Optional.empty());
+    }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org


(james-project) 02/07: [FIX] Cassandra event store: include the snapshot in the event batch

Posted by hq...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

quantranhong1999 pushed a commit to branch 3.9.x
in repository https://gitbox.apache.org/repos/asf/james-project.git

commit e5ac3864fb77f829124bd26e0862c82cf5c5ddff
Author: Benoit TELLIER <bt...@linagora.com>
AuthorDate: Sun Aug 16 16:25:34 2026 +0700

    [FIX] Cassandra event store: include the snapshot in the event batch
---
 .../eventstore/cassandra/EventStoreDao.scala             | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/event-sourcing/event-store-cassandra/src/main/scala/org/apache/james/eventsourcing/eventstore/cassandra/EventStoreDao.scala b/event-sourcing/event-store-cassandra/src/main/scala/org/apache/james/eventsourcing/eventstore/cassandra/EventStoreDao.scala
index b2d7f11f16..63f013d925 100644
--- a/event-sourcing/event-store-cassandra/src/main/scala/org/apache/james/eventsourcing/eventstore/cassandra/EventStoreDao.scala
+++ b/event-sourcing/event-store-cassandra/src/main/scala/org/apache/james/eventsourcing/eventstore/cassandra/EventStoreDao.scala
@@ -89,20 +89,16 @@ class EventStoreDao @Inject() (val session: CqlSession,
       .build())
 
   private[cassandra] def appendAll(events: Iterable[Event], lastSnapShot: Option[EventId]): SMono[Boolean] =
-    SMono(cassandraAsyncExecutor.executeReturnApplied(appendQuery(events))
+    SMono(cassandraAsyncExecutor.executeReturnApplied(appendQuery(events, lastSnapShot))
       .map(_.booleanValue()))
-      .flatMap((success: Boolean) => lastSnapShot
-        .filter(_ => success)
-        .map(id => SMono(cassandraAsyncExecutor.executeVoid(insertSnapshot(events.head.getAggregateId, id))))
-        .getOrElse(SMono.empty)
-        .`then`(SMono.just(success)))
-
-  private def appendQuery(events: Iterable[Event]): Statement[_] =
-    if (events.size == 1)
+
+  private def appendQuery(events: Iterable[Event], lastSnapShot: Option[EventId]): Statement[_] =
+    if (events.size == 1 && lastSnapShot.isEmpty)
       insertEvent(events.head)
     else {
       val batch: BatchStatementBuilder = new BatchStatementBuilder(BatchType.LOGGED)
       events.foreach((event: Event) => batch.addStatement(insertEvent(event)))
+      lastSnapShot.foreach(snapshotId => batch.addStatement(insertSnapshot(events.head.getAggregateId, snapshotId)))
       batch.build()
     }
 
@@ -150,4 +146,4 @@ class EventStoreDao @Inject() (val session: CqlSession,
 
   private def toEvent(row: Row): SMono[Event] = SMono.fromCallable(() => jsonEventSerializer.deserialize(row.get(0, TypeCodecs.TEXT)))
     .subscribeOn(ReactorUtils.BLOCKING_CALL_WRAPPER)
-}
\ No newline at end of file
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org