You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@james.apache.org by bt...@apache.org on 2022/09/12 15:33:37 UTC
[james-project] branch master updated: JAMES-3755 Add a classifiedAsSpam parameter to Rspamd routes (#1172)
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
The following commit(s) were added to refs/heads/master by this push:
new bff0c01358 JAMES-3755 Add a classifiedAsSpam parameter to Rspamd routes (#1172)
bff0c01358 is described below
commit bff0c01358cac99aa6d5cafbdf98e1f16132c808
Author: Benoit TELLIER <bt...@linagora.com>
AuthorDate: Mon Sep 12 17:33:31 2022 +0200
JAMES-3755 Add a classifiedAsSpam parameter to Rspamd routes (#1172)
This allows for instance reporting messages manually reported by the user to the
Spam mailbox with re-reporting messages already classified as Spam by Rspamd,
avoiding creating a positive retro-action loop, and only reporting relevant messages.
Similar for Ham reporting.
---
third-party/rspamd/README.md | 4 +
.../james/rspamd/route/FeedMessageRoute.java | 12 +-
.../james/rspamd/task/FeedHamToRspamdTask.java | 2 +-
.../james/rspamd/task/FeedHamToRspamdTaskDTO.java | 15 +-
.../james/rspamd/task/FeedSpamToRspamdTask.java | 2 +-
.../james/rspamd/task/FeedSpamToRspamdTaskDTO.java | 15 +-
.../rspamd/task/GetMailboxMessagesService.java | 40 +++--
.../apache/james/rspamd/task/RunningOptions.java | 60 ++++++-
.../james/rspamd/route/FeedMessageRouteTest.java | 56 ++++++
...amToRspamdTaskAdditionalInformationDTOTest.java | 20 ++-
.../james/rspamd/task/FeedHamToRspamdTaskTest.java | 187 +++++++++++++++++++-
...amToRspamdTaskAdditionalInformationDTOTest.java | 20 ++-
.../rspamd/task/FeedSpamToRspamdTaskTest.java | 188 ++++++++++++++++++++-
...edHamClassifiedAsHam.additionalInformation.json | 12 ++
...dSpamClassifiedAsHam.additionalInformation.json | 12 ++
15 files changed, 601 insertions(+), 44 deletions(-)
diff --git a/third-party/rspamd/README.md b/third-party/rspamd/README.md
index 0191434e68..296af9e9be 100644
--- a/third-party/rspamd/README.md
+++ b/third-party/rspamd/README.md
@@ -111,6 +111,8 @@ all messages are reported.
These inputs represent the same duration: `1d`, `1day`, `86400 seconds`, `86400`...
- `samplingProbability` (optional): float between 0 and 1, represent the chance to report each given message to Rspamd.
By default, all messages are reported.
+- `classifiedAsSpam` (optional): Boolean, true to only include messages tagged as Spam by Rspamd, false for only
+messages tagged as ham by Rspamd. If omitted all messages are included.
Will return the task id. E.g:
```
@@ -157,6 +159,8 @@ This endpoint has the following param:
These inputs represent the same duration: `1d`, `1day`, `86400 seconds`, `86400`...
- `samplingProbability` (optional): float between 0 and 1, represent the chance to report each given message to Rspamd.
By default, all messages are reported.
+- `classifiedAsSpam` (optional): Boolean, true to only include messages tagged as Spam by Rspamd, false for only
+messages tagged as ham by Rspamd. If omitted all messages are included.
Will return the task id. E.g:
```
diff --git a/third-party/rspamd/src/main/java/org/apache/james/rspamd/route/FeedMessageRoute.java b/third-party/rspamd/src/main/java/org/apache/james/rspamd/route/FeedMessageRoute.java
index ab2e59612f..962b0a1143 100644
--- a/third-party/rspamd/src/main/java/org/apache/james/rspamd/route/FeedMessageRoute.java
+++ b/third-party/rspamd/src/main/java/org/apache/james/rspamd/route/FeedMessageRoute.java
@@ -99,7 +99,8 @@ public class FeedMessageRoute implements Routes {
Optional<Long> periodInSecond = getPeriod(request);
int messagesPerSecond = getMessagesPerSecond(request).orElse(RunningOptions.DEFAULT_MESSAGES_PER_SECOND);
double samplingProbability = getSamplingProbability(request).orElse(RunningOptions.DEFAULT_SAMPLING_PROBABILITY);
- return new RunningOptions(periodInSecond, messagesPerSecond, samplingProbability);
+ Optional<Boolean> classifiedAsSpam = getClassifiedAsSpam(request);
+ return new RunningOptions(periodInSecond, messagesPerSecond, samplingProbability, classifiedAsSpam);
}
private Optional<Long> getPeriod(Request req) {
@@ -140,5 +141,14 @@ public class FeedMessageRoute implements Routes {
throw new IllegalArgumentException("'samplingProbability' must be numeric");
}
}
+
+ private Optional<Boolean> getClassifiedAsSpam(Request req) {
+ try {
+ return Optional.ofNullable(req.queryParams("classifiedAsSpam"))
+ .map(Boolean::parseBoolean);
+ } catch (NumberFormatException ex) {
+ throw new IllegalArgumentException("'classifiedAsSpam' must be a boolean (true|false)");
+ }
+ }
}
diff --git a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedHamToRspamdTask.java b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedHamToRspamdTask.java
index 6f489af400..f0b687e282 100644
--- a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedHamToRspamdTask.java
+++ b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedHamToRspamdTask.java
@@ -237,7 +237,7 @@ public class FeedHamToRspamdTask implements Task {
public Result run() {
Optional<Date> afterDate = runningOptions.getPeriodInSecond().map(periodInSecond -> Date.from(clock.instant().minusSeconds(periodInSecond)));
try {
- return messagesService.getHamMessagesOfAllUser(afterDate, runningOptions.getSamplingProbability(), context)
+ return messagesService.getHamMessagesOfAllUser(afterDate, runningOptions, context)
.transform(ReactorUtils.<MessageResult, Result>throttle()
.elements(runningOptions.getMessagesPerSecond())
.per(Duration.ofSeconds(1))
diff --git a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskDTO.java b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskDTO.java
index c42baa896e..c4f9734033 100644
--- a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskDTO.java
+++ b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskDTO.java
@@ -49,12 +49,14 @@ public class FeedHamToRspamdTaskDTO implements TaskDTO {
rspamdHttpClient,
new RunningOptions(Optional.ofNullable(dto.getPeriodInSecond()),
dto.getMessagesPerSecond(),
- dto.getSamplingProbability()),
+ dto.getSamplingProbability(),
+ dto.getClassifiedAsSpam()),
clock))
.toDTOConverter((domain, type) -> new FeedHamToRspamdTaskDTO(type,
domain.getRunningOptions().getPeriodInSecond().orElse(null),
domain.getRunningOptions().getMessagesPerSecond(),
- domain.getRunningOptions().getSamplingProbability()))
+ domain.getRunningOptions().getSamplingProbability(),
+ domain.getRunningOptions().getClassifiedAsSpam()))
.typeName(FeedHamToRspamdTask.TASK_TYPE.asString())
.withFactory(TaskDTOModule::new);
}
@@ -64,15 +66,18 @@ public class FeedHamToRspamdTaskDTO implements TaskDTO {
private final Long periodInSecond;
private final int messagesPerSecond;
private final double samplingProbability;
+ private final Optional<Boolean> classifiedAsSpam;
public FeedHamToRspamdTaskDTO(@JsonProperty("type") String type,
@JsonProperty("periodInSecond") Long periodInSecond,
@JsonProperty("messagesPerSecond") int messagesPerSecond,
- @JsonProperty("samplingProbability") double samplingProbability) {
+ @JsonProperty("samplingProbability") double samplingProbability,
+ @JsonProperty("classifiedAsSpam") Optional<Boolean> classifiedAsSpam) {
this.type = type;
this.periodInSecond = periodInSecond;
this.messagesPerSecond = messagesPerSecond;
this.samplingProbability = samplingProbability;
+ this.classifiedAsSpam = classifiedAsSpam;
}
@Override
@@ -80,6 +85,10 @@ public class FeedHamToRspamdTaskDTO implements TaskDTO {
return type;
}
+ public Optional<Boolean> getClassifiedAsSpam() {
+ return classifiedAsSpam;
+ }
+
public Long getPeriodInSecond() {
return periodInSecond;
}
diff --git a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedSpamToRspamdTask.java b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedSpamToRspamdTask.java
index 9a33b7440c..aae57cb5f5 100644
--- a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedSpamToRspamdTask.java
+++ b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedSpamToRspamdTask.java
@@ -238,7 +238,7 @@ public class FeedSpamToRspamdTask implements Task {
public Result run() {
Optional<Date> afterDate = runningOptions.getPeriodInSecond().map(periodInSecond -> Date.from(clock.instant().minusSeconds(periodInSecond)));
try {
- return messagesService.getMailboxMessagesOfAllUser(SPAM_MAILBOX_NAME, afterDate, runningOptions.getSamplingProbability(), context)
+ return messagesService.getMailboxMessagesOfAllUser(SPAM_MAILBOX_NAME, afterDate, runningOptions, context)
.transform(ReactorUtils.<MessageResult, Task.Result>throttle()
.elements(runningOptions.getMessagesPerSecond())
.per(Duration.ofSeconds(1))
diff --git a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskDTO.java b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskDTO.java
index 29ce342a07..7ddd635e7e 100644
--- a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskDTO.java
+++ b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskDTO.java
@@ -49,13 +49,15 @@ public class FeedSpamToRspamdTaskDTO implements TaskDTO {
rspamdHttpClient,
new RunningOptions(Optional.ofNullable(dto.getPeriodInSecond()),
dto.getMessagesPerSecond(),
- dto.getSamplingProbability()),
+ dto.getSamplingProbability(),
+ dto.getClassifiedAsSpam()),
clock))
.toDTOConverter((domain, type) -> new FeedSpamToRspamdTaskDTO(
type,
domain.getRunningOptions().getPeriodInSecond().orElse(null),
domain.getRunningOptions().getMessagesPerSecond(),
- domain.getRunningOptions().getSamplingProbability()))
+ domain.getRunningOptions().getSamplingProbability(),
+ domain.getRunningOptions().getClassifiedAsSpam()))
.typeName(FeedSpamToRspamdTask.TASK_TYPE.asString())
.withFactory(TaskDTOModule::new);
}
@@ -64,16 +66,19 @@ public class FeedSpamToRspamdTaskDTO implements TaskDTO {
private final Long periodInSecond;
private final int messagesPerSecond;
private final double samplingProbability;
+ private final Optional<Boolean> classifiedAsSpam;
public FeedSpamToRspamdTaskDTO(@JsonProperty("type") String type,
@JsonProperty("periodInSecond") Long periodInSecond,
@JsonProperty("messagesPerSecond") int messagesPerSecond,
- @JsonProperty("samplingProbability") double samplingProbability) {
+ @JsonProperty("samplingProbability") double samplingProbability,
+ @JsonProperty("classifiedAsSpam") Optional<Boolean> classifiedAsSpam) {
this.type = type;
this.periodInSecond = periodInSecond;
this.messagesPerSecond = messagesPerSecond;
this.samplingProbability = samplingProbability;
+ this.classifiedAsSpam = classifiedAsSpam;
}
@Override
@@ -92,4 +97,8 @@ public class FeedSpamToRspamdTaskDTO implements TaskDTO {
public double getSamplingProbability() {
return samplingProbability;
}
+
+ public Optional<Boolean> getClassifiedAsSpam() {
+ return classifiedAsSpam;
+ }
}
\ No newline at end of file
diff --git a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/GetMailboxMessagesService.java b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/GetMailboxMessagesService.java
index 95ba0550c8..1e00b58434 100644
--- a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/GetMailboxMessagesService.java
+++ b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/GetMailboxMessagesService.java
@@ -30,9 +30,11 @@ import org.apache.james.mailbox.MailboxSession;
import org.apache.james.mailbox.MessageIdManager;
import org.apache.james.mailbox.MessageManager;
import org.apache.james.mailbox.model.FetchGroup;
+import org.apache.james.mailbox.model.MailboxMetaData;
import org.apache.james.mailbox.model.MailboxPath;
import org.apache.james.mailbox.model.MessageRange;
import org.apache.james.mailbox.model.MessageResult;
+import org.apache.james.mailbox.model.search.MailboxQuery;
import org.apache.james.mailbox.store.MailboxSessionMapperFactory;
import org.apache.james.mailbox.store.mail.MessageMapper;
import org.apache.james.mailbox.store.mail.model.Message;
@@ -62,22 +64,24 @@ public class GetMailboxMessagesService {
this.messageIdManager = messageIdManager;
}
- public Flux<MessageResult> getMailboxMessagesOfAllUser(String mailboxName, Optional<Date> afterDate, double samplingProbability,
+ public Flux<MessageResult> getMailboxMessagesOfAllUser(String mailboxName, Optional<Date> afterDate, RunningOptions runningOptions,
FeedSpamToRspamdTask.Context context) throws UsersRepositoryException {
return Iterators.toFlux(userRepository.list())
- .flatMap(username -> getMailboxMessagesOfAUser(username, mailboxName, afterDate, samplingProbability, context), ReactorUtils.DEFAULT_CONCURRENCY);
+ .flatMap(username -> getMailboxMessagesOfAUser(username, mailboxName, afterDate, runningOptions, context), ReactorUtils.DEFAULT_CONCURRENCY);
}
- public Flux<MessageResult> getHamMessagesOfAllUser(Optional<Date> afterDate, double samplingProbability,
+ public Flux<MessageResult> getHamMessagesOfAllUser(Optional<Date> afterDate, RunningOptions runningOptions,
FeedHamToRspamdTask.Context context) throws UsersRepositoryException {
return Iterators.toFlux(userRepository.list())
- .flatMap(Throwing.function(username -> Flux.fromIterable(mailboxManager.list(mailboxManager.createSystemSession(username)))
- .filter(this::hamMailboxesPredicate)
- .flatMap(mailboxPath -> getMailboxMessagesOfAUser(username, mailboxPath, afterDate, samplingProbability, context), 2)), ReactorUtils.DEFAULT_CONCURRENCY);
+ .flatMap(Throwing.function(username ->
+ Flux.from(mailboxManager.search(MailboxQuery.privateMailboxesBuilder(mailboxManager.createSystemSession(username)).build(),
+ mailboxManager.createSystemSession(username)))
+ .filter(mbxMetadata -> hamMailboxesPredicate(mbxMetadata.getPath()))
+ .flatMap(mbxMetadata -> getMailboxMessagesOfAUser(username, mbxMetadata, afterDate, runningOptions, context), 2)), ReactorUtils.DEFAULT_CONCURRENCY);
}
private Flux<MessageResult> getMailboxMessagesOfAUser(Username username, String mailboxName, Optional<Date> afterDate,
- double samplingProbability, FeedSpamToRspamdTask.Context context) {
+ RunningOptions runningOptions, FeedSpamToRspamdTask.Context context) {
MailboxSession mailboxSession = mailboxManager.createSystemSession(username);
return Mono.from(mailboxManager.getMailboxReactive(MailboxPath.forUser(username, mailboxName), mailboxSession))
@@ -85,32 +89,34 @@ public class GetMailboxMessagesService {
.flatMapMany(Throwing.function(mailbox -> mapperFactory.getMessageMapper(mailboxSession).findInMailboxReactive(mailbox, MessageRange.all(), MessageMapper.FetchType.METADATA, UNLIMITED)))
.doOnNext(mailboxMessageMetaData -> context.incrementSpamMessageCount())
.filter(mailboxMessageMetaData -> afterDate.map(date -> mailboxMessageMetaData.getInternalDate().after(date)).orElse(true))
- .filter(message -> randomBooleanWithProbability(samplingProbability))
+ .filter(message -> randomBooleanWithProbability(runningOptions))
.map(Message::getMessageId)
.collectList()
- .flatMapMany(messageIds -> messageIdManager.getMessagesReactive(messageIds, FetchGroup.FULL_CONTENT, mailboxSession));
+ .flatMapMany(messageIds -> messageIdManager.getMessagesReactive(messageIds, FetchGroup.FULL_CONTENT, mailboxSession))
+ .filter(runningOptions.correspondingClassificationFilter());
}
- private Flux<MessageResult> getMailboxMessagesOfAUser(Username username, MailboxPath mailboxPath, Optional<Date> afterDate,
- double samplingProbability, FeedHamToRspamdTask.Context context) {
+ private Flux<MessageResult> getMailboxMessagesOfAUser(Username username, MailboxMetaData mailboxMetaData, Optional<Date> afterDate,
+ RunningOptions runningOptions, FeedHamToRspamdTask.Context context) {
MailboxSession mailboxSession = mailboxManager.createSystemSession(username);
- return Mono.from(mailboxManager.getMailboxReactive(mailboxPath, mailboxSession))
+ return Mono.from(mailboxManager.getMailboxReactive(mailboxMetaData.getId(), mailboxSession))
.map(Throwing.function(MessageManager::getMailboxEntity))
.flatMapMany(Throwing.function(mailbox -> mapperFactory.getMessageMapper(mailboxSession).findInMailboxReactive(mailbox, MessageRange.all(), MessageMapper.FetchType.METADATA, UNLIMITED)))
.doOnNext(mailboxMessageMetaData -> context.incrementHamMessageCount())
.filter(mailboxMessageMetaData -> afterDate.map(date -> mailboxMessageMetaData.getInternalDate().after(date)).orElse(true))
- .filter(message -> randomBooleanWithProbability(samplingProbability))
+ .filter(message -> randomBooleanWithProbability(runningOptions))
.map(Message::getMessageId)
.collectList()
- .flatMapMany(messageIds -> messageIdManager.getMessagesReactive(messageIds, FetchGroup.FULL_CONTENT, mailboxSession));
+ .flatMapMany(messageIds -> messageIdManager.getMessagesReactive(messageIds, FetchGroup.FULL_CONTENT, mailboxSession))
+ .filter(runningOptions.correspondingClassificationFilter());
}
- public static boolean randomBooleanWithProbability(double probability) {
- if (probability == 1.0) {
+ public static boolean randomBooleanWithProbability(RunningOptions runningOptions) {
+ if (runningOptions.getSamplingProbability() == 1.0) {
return true;
}
- return Math.random() < probability;
+ return Math.random() < runningOptions.getSamplingProbability();
}
private boolean hamMailboxesPredicate(MailboxPath mailboxPath) {
diff --git a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/RunningOptions.java b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/RunningOptions.java
index bc354c9511..e808c922ee 100644
--- a/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/RunningOptions.java
+++ b/third-party/rspamd/src/main/java/org/apache/james/rspamd/task/RunningOptions.java
@@ -20,25 +20,70 @@
package org.apache.james.rspamd.task;
import java.util.Optional;
+import java.util.function.Predicate;
+import org.apache.james.mailbox.exception.MailboxException;
+import org.apache.james.mailbox.model.MessageResult;
+import org.apache.james.rspamd.RspamdScanner;
+import org.apache.james.util.streams.Iterators;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
public class RunningOptions {
+ interface ClassificationFilter extends Predicate<MessageResult> {
+ ClassificationFilter ALL = any -> true;
+ ClassificationFilter CLASSIFIED_AS_HAM = new HeaderBasedPredicate("NO");
+ ClassificationFilter CLASSIFIED_AS_SPAM = new HeaderBasedPredicate("YES");
+
+ class HeaderBasedPredicate implements ClassificationFilter {
+ private final String value;
+
+ public HeaderBasedPredicate(String value) {
+ this.value = value;
+ }
+
+ @Override
+ public boolean test(MessageResult messageResult) {
+ try {
+ return Iterators.toStream(messageResult.getHeaders().headers())
+ .filter(header -> header.getName().equalsIgnoreCase(RspamdScanner.FLAG_MAIL.asString()))
+ .findFirst()
+ .map(header -> header.getValue().equalsIgnoreCase(value))
+ // Message was not classified by Rspamd, include it
+ .orElse(true);
+ } catch (MailboxException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+
+
+
public static final Optional<Long> DEFAULT_PERIOD = Optional.empty();
public static final int DEFAULT_MESSAGES_PER_SECOND = 10;
public static final double DEFAULT_SAMPLING_PROBABILITY = 1;
- public static final RunningOptions DEFAULT = new RunningOptions(DEFAULT_PERIOD, DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY);
+ public static final Optional<Boolean> ALL_MESSAGES = Optional.empty();
+ public static final RunningOptions DEFAULT = new RunningOptions(DEFAULT_PERIOD, DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY, ALL_MESSAGES);
private final Optional<Long> periodInSecond;
private final int messagesPerSecond;
private final double samplingProbability;
+ private final Optional<Boolean> classifiedAsSpam;
public RunningOptions(@JsonProperty("periodInSecond") Optional<Long> periodInSecond,
@JsonProperty("messagesPerSecond") int messagesPerSecond,
- @JsonProperty("samplingProbability") double samplingProbability) {
+ @JsonProperty("samplingProbability") double samplingProbability,
+ @JsonProperty("classifiedAsSpam") Optional<Boolean> classifiedAsSpam) {
this.periodInSecond = periodInSecond;
this.messagesPerSecond = messagesPerSecond;
this.samplingProbability = samplingProbability;
+ this.classifiedAsSpam = classifiedAsSpam;
+ }
+
+ public Optional<Boolean> getClassifiedAsSpam() {
+ return classifiedAsSpam;
}
public Optional<Long> getPeriodInSecond() {
@@ -52,4 +97,15 @@ public class RunningOptions {
public double getSamplingProbability() {
return samplingProbability;
}
+
+ @JsonIgnore
+ public ClassificationFilter correspondingClassificationFilter() {
+ return classifiedAsSpam.map(result -> {
+ if (result) {
+ return ClassificationFilter.CLASSIFIED_AS_SPAM;
+ } else {
+ return ClassificationFilter.CLASSIFIED_AS_HAM;
+ }
+ }).orElse(ClassificationFilter.ALL);
+ }
}
diff --git a/third-party/rspamd/src/test/java/org/apache/james/rspamd/route/FeedMessageRouteTest.java b/third-party/rspamd/src/test/java/org/apache/james/rspamd/route/FeedMessageRouteTest.java
index cc45f95d75..c4bd01fb4a 100644
--- a/third-party/rspamd/src/test/java/org/apache/james/rspamd/route/FeedMessageRouteTest.java
+++ b/third-party/rspamd/src/test/java/org/apache/james/rspamd/route/FeedMessageRouteTest.java
@@ -181,6 +181,34 @@ public class FeedMessageRouteTest {
.body("additionalInformation.runningOptions.samplingProbability", is((float) RunningOptions.DEFAULT_SAMPLING_PROBABILITY));
}
+ @Test
+ void taskShouldDisplayClassifiedAsSpamRunningOption() throws MailboxException {
+ appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW));
+ appendMessage(ALICE_SPAM_MAILBOX, Date.from(NOW));
+
+ String taskId = given()
+ .queryParam("action", "reportSpam")
+ .queryParam("classifiedAsSpam", "false")
+ .post()
+ .jsonPath()
+ .get("taskId");
+
+ given()
+ .basePath(TasksRoutes.BASE)
+ .when()
+ .get(taskId + "/await")
+ .then()
+ .body("status", is("completed"))
+ .body("additionalInformation.type", is(FeedSpamToRspamdTask.TASK_TYPE.asString()))
+ .body("additionalInformation.spamMessageCount", is(2))
+ .body("additionalInformation.reportedSpamMessageCount", is(2))
+ .body("additionalInformation.errorCount", is(0))
+ .body("additionalInformation.runningOptions.classifiedAsSpam", is(false))
+ .body("additionalInformation.runningOptions.messagesPerSecond", is(RunningOptions.DEFAULT_MESSAGES_PER_SECOND))
+ .body("additionalInformation.runningOptions.periodInSecond", is(nullValue()))
+ .body("additionalInformation.runningOptions.samplingProbability", is((float) RunningOptions.DEFAULT_SAMPLING_PROBABILITY));
+ }
+
@Test
void taskShouldReportOnlyMailInPeriod() throws MailboxException {
appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(THREE_DAYS_IN_SECOND)));
@@ -429,6 +457,34 @@ public class FeedMessageRouteTest {
.body("additionalInformation.runningOptions.samplingProbability", is((float) RunningOptions.DEFAULT_SAMPLING_PROBABILITY));
}
+ @Test
+ void taskShouldDisplayClassifiedAsSpamRunningOption() throws MailboxException {
+ appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW));
+ appendMessage(ALICE_INBOX_MAILBOX, Date.from(NOW));
+
+ String taskId = given()
+ .queryParam("action", "reportHam")
+ .queryParam("classifiedAsSpam", "true")
+ .post()
+ .jsonPath()
+ .get("taskId");
+
+ given()
+ .basePath(TasksRoutes.BASE)
+ .when()
+ .get(taskId + "/await")
+ .then()
+ .body("status", is("completed"))
+ .body("additionalInformation.type", is(FeedHamToRspamdTask.TASK_TYPE.asString()))
+ .body("additionalInformation.hamMessageCount", is(2))
+ .body("additionalInformation.reportedHamMessageCount", is(2))
+ .body("additionalInformation.errorCount", is(0))
+ .body("additionalInformation.runningOptions.classifiedAsSpam", is(true))
+ .body("additionalInformation.runningOptions.messagesPerSecond", is(RunningOptions.DEFAULT_MESSAGES_PER_SECOND))
+ .body("additionalInformation.runningOptions.periodInSecond", is(nullValue()))
+ .body("additionalInformation.runningOptions.samplingProbability", is((float) RunningOptions.DEFAULT_SAMPLING_PROBABILITY));
+ }
+
@Test
void taskShouldReportOnlyMailInPeriod() throws MailboxException {
appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(THREE_DAYS_IN_SECOND)));
diff --git a/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskAdditionalInformationDTOTest.java b/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskAdditionalInformationDTOTest.java
index 2705eca987..d9fd376b15 100644
--- a/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskAdditionalInformationDTOTest.java
+++ b/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskAdditionalInformationDTOTest.java
@@ -40,6 +40,23 @@ class FeedHamToRspamdTaskAdditionalInformationDTOTest {
.verify();
}
+ @Test
+ void shouldMatchJsonSerializationContractWhenClassifiedAsHam() throws Exception {
+ JsonSerializationVerifier.dtoModule(FeedHamToRspamdTaskAdditionalInformationDTO.SERIALIZATION_MODULE)
+ .bean(new FeedHamToRspamdTask.AdditionalInformation(
+ Instant.parse("2007-12-03T10:15:30.00Z"),
+ 4,
+ 2,
+ 1,
+ new RunningOptions(
+ Optional.empty(),
+ RunningOptions.DEFAULT_MESSAGES_PER_SECOND,
+ RunningOptions.DEFAULT_SAMPLING_PROBABILITY,
+ Optional.of(false))))
+ .json(ClassLoaderUtils.getSystemResourceAsString("json/feedHamClassifiedAsHam.additionalInformation.json"))
+ .verify();
+ }
+
@Test
void shouldMatchJsonSerializationContractWhenNonEmptyPeriod() throws Exception {
JsonSerializationVerifier.dtoModule(FeedHamToRspamdTaskAdditionalInformationDTO.SERIALIZATION_MODULE)
@@ -51,7 +68,8 @@ class FeedHamToRspamdTaskAdditionalInformationDTOTest {
new RunningOptions(
Optional.of(3600L),
RunningOptions.DEFAULT_MESSAGES_PER_SECOND,
- RunningOptions.DEFAULT_SAMPLING_PROBABILITY)))
+ RunningOptions.DEFAULT_SAMPLING_PROBABILITY,
+ Optional.empty())))
.json(ClassLoaderUtils.getSystemResourceAsString("json/feedHamNonEmptyPeriod.additionalInformation.json"))
.verify();
}
diff --git a/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskTest.java b/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskTest.java
index 65a187e9dc..0d9a2d8dd2 100644
--- a/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskTest.java
+++ b/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedHamToRspamdTaskTest.java
@@ -21,6 +21,7 @@ package org.apache.james.rspamd.task;
import static org.apache.james.rspamd.DockerRspamd.PASSWORD;
import static org.apache.james.rspamd.task.FeedSpamToRspamdTaskTest.BOB_SPAM_MAILBOX;
+import static org.apache.james.rspamd.task.RunningOptions.ALL_MESSAGES;
import static org.apache.james.rspamd.task.RunningOptions.DEFAULT_MESSAGES_PER_SECOND;
import static org.apache.james.rspamd.task.RunningOptions.DEFAULT_SAMPLING_PROBABILITY;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
@@ -144,7 +145,7 @@ public class FeedHamToRspamdTaskTest {
@Test
void taskShouldReportHamMessageInPeriod() throws MailboxException {
RunningOptions runningOptions = new RunningOptions(Optional.of(TWO_DAYS_IN_SECOND),
- DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY);
+ DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY, ALL_MESSAGES);
task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
appendHamMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)));
@@ -163,7 +164,7 @@ public class FeedHamToRspamdTaskTest {
@Test
void taskShouldNotReportHamMessageNotInPeriod() throws MailboxException {
RunningOptions runningOptions = new RunningOptions(Optional.of(TWO_DAYS_IN_SECOND),
- DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY);
+ DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY, ALL_MESSAGES);
task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
appendHamMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(THREE_DAYS_IN_SECOND)));
@@ -182,7 +183,7 @@ public class FeedHamToRspamdTaskTest {
@Test
void mixedInternalDateCase() throws MailboxException {
RunningOptions runningOptions = new RunningOptions(Optional.of(TWO_DAYS_IN_SECOND),
- DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY);
+ DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY, ALL_MESSAGES);
task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
appendHamMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(THREE_DAYS_IN_SECOND)));
@@ -202,7 +203,7 @@ public class FeedHamToRspamdTaskTest {
@Test
void taskWithSamplingProbabilityIsZeroShouldReportNonHamMessage() {
RunningOptions runningOptions = new RunningOptions(Optional.empty(),
- DEFAULT_MESSAGES_PER_SECOND, 0);
+ DEFAULT_MESSAGES_PER_SECOND, 0, ALL_MESSAGES);
task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
IntStream.range(0, 10)
@@ -238,7 +239,7 @@ public class FeedHamToRspamdTaskTest {
@Test
void taskWithVeryLowSamplingProbabilityShouldReportNotAllHamMessages() {
RunningOptions runningOptions = new RunningOptions(Optional.empty(),
- DEFAULT_MESSAGES_PER_SECOND, 0.01);
+ DEFAULT_MESSAGES_PER_SECOND, 0.01, ALL_MESSAGES);
task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
IntStream.range(0, 10)
@@ -257,7 +258,7 @@ public class FeedHamToRspamdTaskTest {
@Test
void taskWithVeryHighSamplingProbabilityShouldReportMoreThanZeroMessage() {
RunningOptions runningOptions = new RunningOptions(Optional.empty(),
- DEFAULT_MESSAGES_PER_SECOND, 0.99);
+ DEFAULT_MESSAGES_PER_SECOND, 0.99, ALL_MESSAGES);
task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
IntStream.range(0, 10)
@@ -276,7 +277,7 @@ public class FeedHamToRspamdTaskTest {
@Test
void taskWithAverageSamplingProbabilityShouldReportSomeMessages() {
RunningOptions runningOptions = new RunningOptions(Optional.empty(),
- DEFAULT_MESSAGES_PER_SECOND, 0.5);
+ DEFAULT_MESSAGES_PER_SECOND, 0.5, ALL_MESSAGES);
task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
IntStream.range(0, 10)
@@ -292,6 +293,168 @@ public class FeedHamToRspamdTaskTest {
});
}
+ @Test
+ void shouldReportUnclassifiedWhenClassifiedAsSpamIsTrue() throws Exception {
+ RunningOptions runningOptions = new RunningOptions(Optional.empty(),
+ DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(true));
+ task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
+
+ appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "Unrelated: at all");
+
+ Task.Result result = task.run();
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ assertThat(task.snapshot().getHamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getReportedHamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getErrorCount()).isZero();
+ });
+ }
+
+ @Test
+ void shouldNotReportHamWhenClassifiedAsSpamIsTrue() throws Exception {
+ RunningOptions runningOptions = new RunningOptions(Optional.empty(),
+ DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(true));
+ task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
+
+ appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: NO");
+
+ Task.Result result = task.run();
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ assertThat(task.snapshot().getHamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getReportedHamMessageCount()).isZero();
+ assertThat(task.snapshot().getErrorCount()).isZero();
+ });
+ }
+
+ @Test
+ void shouldReportSpamWhenClassifiedAsSpamIsTrue() throws Exception {
+ RunningOptions runningOptions = new RunningOptions(Optional.empty(),
+ DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(true));
+ task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
+
+ appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: YES");
+
+ Task.Result result = task.run();
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ assertThat(task.snapshot().getHamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getReportedHamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getErrorCount()).isZero();
+ });
+ }
+
+ @Test
+ void shouldReportUnclassifiedWhenClassifiedAsSpamIsOmited() throws Exception {
+ RunningOptions runningOptions = new RunningOptions(Optional.empty(),
+ DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.empty());
+ task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
+
+ appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "Unrelated: at all");
+
+ Task.Result result = task.run();
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ assertThat(task.snapshot().getHamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getReportedHamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getErrorCount()).isZero();
+ });
+ }
+
+ @Test
+ void shouldReportHamWhenClassifiedAsSpamIsOmited() throws Exception {
+ RunningOptions runningOptions = new RunningOptions(Optional.empty(),
+ DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.empty());
+ task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
+
+ appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: NO");
+
+ Task.Result result = task.run();
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ assertThat(task.snapshot().getHamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getReportedHamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getErrorCount()).isZero();
+ });
+ }
+
+ @Test
+ void shouldNotReportSpamWhenClassifiedAsSpamIsOmited() throws Exception {
+ RunningOptions runningOptions = new RunningOptions(Optional.empty(),
+ DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.empty());
+ task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
+
+ appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: YES");
+
+ Task.Result result = task.run();
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ assertThat(task.snapshot().getHamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getReportedHamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getErrorCount()).isZero();
+ });
+ }
+
+ @Test
+ void shouldReportUnclassifiedWhenClassifiedAsSpamIsFalse() throws Exception {
+ RunningOptions runningOptions = new RunningOptions(Optional.empty(),
+ DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(false));
+ task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
+
+ appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "Unrelated: at all");
+
+ Task.Result result = task.run();
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ assertThat(task.snapshot().getHamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getReportedHamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getErrorCount()).isZero();
+ });
+ }
+
+ @Test
+ void shouldReportHamWhenClassifiedAsSpamIsFalse() throws Exception {
+ RunningOptions runningOptions = new RunningOptions(Optional.empty(),
+ DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(false));
+ task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
+
+ appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: NO");
+
+ Task.Result result = task.run();
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ assertThat(task.snapshot().getHamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getReportedHamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getErrorCount()).isZero();
+ });
+ }
+
+ @Test
+ void shouldNotReportSpamWhenClassifiedAsSpamIsFalse() throws Exception {
+ RunningOptions runningOptions = new RunningOptions(Optional.empty(),
+ DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(false));
+ task = new FeedHamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
+
+ appendMessage(BOB_INBOX_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: YES");
+
+ Task.Result result = task.run();
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ assertThat(task.snapshot().getHamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getReportedHamMessageCount()).isZero();
+ assertThat(task.snapshot().getErrorCount()).isZero();
+ });
+ }
+
@Test
void shouldNotReportMessagesInTrashAndSpamMailboxes() throws MailboxException {
appendHamMessage(BOB_TRASH_MAILBOX, Date.from(NOW));
@@ -351,4 +514,14 @@ public class FeedHamToRspamdTaskTest {
true,
new Flags());
}
+
+ private void appendMessage(MailboxPath mailboxPath, Date internalDate, String header) throws MailboxException {
+ MailboxSession session = mailboxManager.createSystemSession(mailboxPath.getUser());
+ mailboxManager.getMailbox(mailboxPath, session)
+ .appendMessage(new ByteArrayInputStream((header + "\r\n\r\n" + String.format("random content %4.3f", Math.random())).getBytes()),
+ internalDate,
+ session,
+ true,
+ new Flags());
+ }
}
diff --git a/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskAdditionalInformationDTOTest.java b/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskAdditionalInformationDTOTest.java
index 8471affa3a..70ba6ce805 100644
--- a/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskAdditionalInformationDTOTest.java
+++ b/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskAdditionalInformationDTOTest.java
@@ -43,6 +43,23 @@ class FeedSpamToRspamdTaskAdditionalInformationDTOTest {
.verify();
}
+ @Test
+ void shouldMatchJsonSerializationContractWhenClassifiedAsHam() throws Exception {
+ JsonSerializationVerifier.dtoModule(FeedSpamToRspamdTaskAdditionalInformationDTO.SERIALIZATION_MODULE)
+ .bean(new FeedSpamToRspamdTask.AdditionalInformation(
+ Instant.parse("2007-12-03T10:15:30.00Z"),
+ 4,
+ 2,
+ 1,
+ new RunningOptions(
+ Optional.empty(),
+ RunningOptions.DEFAULT_MESSAGES_PER_SECOND,
+ RunningOptions.DEFAULT_SAMPLING_PROBABILITY,
+ Optional.of(false))))
+ .json(ClassLoaderUtils.getSystemResourceAsString("json/feedSpamClassifiedAsHam.additionalInformation.json"))
+ .verify();
+ }
+
@Test
void shouldMatchJsonSerializationContractWhenNonEmptyPeriod() throws Exception {
JsonSerializationVerifier.dtoModule(FeedSpamToRspamdTaskAdditionalInformationDTO.SERIALIZATION_MODULE)
@@ -54,7 +71,8 @@ class FeedSpamToRspamdTaskAdditionalInformationDTOTest {
new RunningOptions(
Optional.of(3600L),
DEFAULT_MESSAGES_PER_SECOND,
- DEFAULT_SAMPLING_PROBABILITY)))
+ DEFAULT_SAMPLING_PROBABILITY,
+ Optional.empty())))
.json(ClassLoaderUtils.getSystemResourceAsString("json/feedSpamNonEmptyPeriod.additionalInformation.json"))
.verify();
}
diff --git a/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskTest.java b/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskTest.java
index 9f800d73bf..3bd9abd943 100644
--- a/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskTest.java
+++ b/third-party/rspamd/src/test/java/org/apache/james/rspamd/task/FeedSpamToRspamdTaskTest.java
@@ -21,6 +21,7 @@ package org.apache.james.rspamd.task;
import static org.apache.james.rspamd.DockerRspamd.PASSWORD;
import static org.apache.james.rspamd.task.FeedSpamToRspamdTask.SPAM_MAILBOX_NAME;
+import static org.apache.james.rspamd.task.RunningOptions.ALL_MESSAGES;
import static org.apache.james.rspamd.task.RunningOptions.DEFAULT_MESSAGES_PER_SECOND;
import static org.apache.james.rspamd.task.RunningOptions.DEFAULT_SAMPLING_PROBABILITY;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
@@ -137,7 +138,7 @@ public class FeedSpamToRspamdTaskTest {
@Test
void taskShouldReportSpamMessageInPeriod() throws MailboxException {
RunningOptions runningOptions = new RunningOptions(Optional.of(TWO_DAYS_IN_SECOND),
- DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY);
+ DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY, ALL_MESSAGES);
task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
appendSpamMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)));
@@ -156,7 +157,7 @@ public class FeedSpamToRspamdTaskTest {
@Test
void taskShouldNotReportSpamMessageNotInPeriod() throws MailboxException {
RunningOptions runningOptions = new RunningOptions(Optional.of(TWO_DAYS_IN_SECOND),
- DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY);
+ DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY, ALL_MESSAGES);
task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
appendSpamMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(THREE_DAYS_IN_SECOND)));
@@ -175,7 +176,7 @@ public class FeedSpamToRspamdTaskTest {
@Test
void mixedInternalDateCase() throws MailboxException {
RunningOptions runningOptions = new RunningOptions(Optional.of(TWO_DAYS_IN_SECOND),
- DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY);
+ DEFAULT_MESSAGES_PER_SECOND, DEFAULT_SAMPLING_PROBABILITY, ALL_MESSAGES);
task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
appendSpamMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(THREE_DAYS_IN_SECOND)));
@@ -195,7 +196,7 @@ public class FeedSpamToRspamdTaskTest {
@Test
void taskWithSamplingProbabilityIsZeroShouldReportNonSpamMessage() {
RunningOptions runningOptions = new RunningOptions(Optional.empty(),
- DEFAULT_MESSAGES_PER_SECOND, 0);
+ DEFAULT_MESSAGES_PER_SECOND, 0, ALL_MESSAGES);
task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
IntStream.range(0, 10)
@@ -231,7 +232,7 @@ public class FeedSpamToRspamdTaskTest {
@Test
void taskWithVeryLowSamplingProbabilityShouldReportNotAllSpamMessages() {
RunningOptions runningOptions = new RunningOptions(Optional.empty(),
- DEFAULT_MESSAGES_PER_SECOND, 0.01);
+ DEFAULT_MESSAGES_PER_SECOND, 0.01, ALL_MESSAGES);
task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
IntStream.range(0, 10)
@@ -250,7 +251,7 @@ public class FeedSpamToRspamdTaskTest {
@Test
void taskWithVeryHighSamplingProbabilityShouldReportMoreThanZeroMessage() {
RunningOptions runningOptions = new RunningOptions(Optional.empty(),
- DEFAULT_MESSAGES_PER_SECOND, 0.99);
+ DEFAULT_MESSAGES_PER_SECOND, 0.99, ALL_MESSAGES);
task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
IntStream.range(0, 10)
@@ -269,7 +270,7 @@ public class FeedSpamToRspamdTaskTest {
@Test
void taskWithAverageSamplingProbabilityShouldReportSomeMessages() {
RunningOptions runningOptions = new RunningOptions(Optional.empty(),
- DEFAULT_MESSAGES_PER_SECOND, 0.5);
+ DEFAULT_MESSAGES_PER_SECOND, 0.5, ALL_MESSAGES);
task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
IntStream.range(0, 10)
@@ -285,6 +286,169 @@ public class FeedSpamToRspamdTaskTest {
});
}
+ @Test
+ void shouldReportUnclassifiedWhenClassifiedAsSpamIsTrue() throws Exception {
+ RunningOptions runningOptions = new RunningOptions(Optional.empty(),
+ DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(true));
+ task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
+
+ appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "Unrelated: at all");
+
+ Task.Result result = task.run();
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ assertThat(task.snapshot().getSpamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getReportedSpamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getErrorCount()).isZero();
+ });
+ }
+
+ @Test
+ void shouldNotReportHamWhenClassifiedAsSpamIsTrue() throws Exception {
+ RunningOptions runningOptions = new RunningOptions(Optional.empty(),
+ DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(true));
+ task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
+
+ appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: NO");
+
+ Task.Result result = task.run();
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ assertThat(task.snapshot().getSpamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getReportedSpamMessageCount()).isZero();
+ assertThat(task.snapshot().getErrorCount()).isZero();
+ });
+ }
+
+ @Test
+ void shouldReportSpamWhenClassifiedAsSpamIsTrue() throws Exception {
+ RunningOptions runningOptions = new RunningOptions(Optional.empty(),
+ DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(true));
+ task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
+
+ appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: YES");
+
+ Task.Result result = task.run();
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ assertThat(task.snapshot().getSpamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getReportedSpamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getErrorCount()).isZero();
+ });
+ }
+
+ @Test
+ void shouldReportUnclassifiedWhenClassifiedAsSpamIsOmited() throws Exception {
+ RunningOptions runningOptions = new RunningOptions(Optional.empty(),
+ DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.empty());
+ task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
+
+ appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "Unrelated: at all");
+
+ Task.Result result = task.run();
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ assertThat(task.snapshot().getSpamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getReportedSpamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getErrorCount()).isZero();
+ });
+ }
+
+ @Test
+ void shouldReportHamWhenClassifiedAsSpamIsOmited() throws Exception {
+ RunningOptions runningOptions = new RunningOptions(Optional.empty(),
+ DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.empty());
+ task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
+
+ appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: NO");
+
+ Task.Result result = task.run();
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ assertThat(task.snapshot().getSpamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getReportedSpamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getErrorCount()).isZero();
+ });
+ }
+
+ @Test
+ void shouldNotReportSpamWhenClassifiedAsSpamIsOmited() throws Exception {
+ RunningOptions runningOptions = new RunningOptions(Optional.empty(),
+ DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.empty());
+ task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
+
+ appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: YES");
+
+ Task.Result result = task.run();
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ assertThat(task.snapshot().getSpamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getReportedSpamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getErrorCount()).isZero();
+ });
+ }
+
+ @Test
+ void shouldReportUnclassifiedWhenClassifiedAsSpamIsFalse() throws Exception {
+ RunningOptions runningOptions = new RunningOptions(Optional.empty(),
+ DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(false));
+ task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
+
+ appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "Unrelated: at all");
+
+ Task.Result result = task.run();
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ assertThat(task.snapshot().getSpamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getReportedSpamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getErrorCount()).isZero();
+ });
+ }
+
+ @Test
+ void shouldReportHamWhenClassifiedAsSpamIsFalse() throws Exception {
+ RunningOptions runningOptions = new RunningOptions(Optional.empty(),
+ DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(false));
+ task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
+
+ appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: NO");
+
+ Task.Result result = task.run();
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ assertThat(task.snapshot().getSpamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getReportedSpamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getErrorCount()).isZero();
+ });
+ }
+
+ @Test
+ void shouldNotReportSpamWhenClassifiedAsSpamIsFalse() throws Exception {
+ RunningOptions runningOptions = new RunningOptions(Optional.empty(),
+ DEFAULT_MESSAGES_PER_SECOND, 1.0, Optional.of(false));
+ task = new FeedSpamToRspamdTask(mailboxManager, usersRepository, messageIdManager, mapperFactory, client, runningOptions, clock);
+
+ appendMessage(BOB_SPAM_MAILBOX, Date.from(NOW.minusSeconds(ONE_DAY_IN_SECOND)), "org.apache.james.rspamd.flag: YES");
+
+ Task.Result result = task.run();
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ assertThat(task.snapshot().getSpamMessageCount()).isEqualTo(1);
+ assertThat(task.snapshot().getReportedSpamMessageCount()).isZero();
+ assertThat(task.snapshot().getErrorCount()).isZero();
+ });
+ }
+
+
private void appendSpamMessage(MailboxPath mailboxPath, Date internalDate) throws MailboxException {
MailboxSession session = mailboxManager.createSystemSession(mailboxPath.getUser());
mailboxManager.getMailbox(mailboxPath, session)
@@ -294,4 +458,14 @@ public class FeedSpamToRspamdTaskTest {
true,
new Flags());
}
+
+ private void appendMessage(MailboxPath mailboxPath, Date internalDate, String header) throws MailboxException {
+ MailboxSession session = mailboxManager.createSystemSession(mailboxPath.getUser());
+ mailboxManager.getMailbox(mailboxPath, session)
+ .appendMessage(new ByteArrayInputStream((header + "\r\n\r\n" + String.format("random content %4.3f", Math.random())).getBytes()),
+ internalDate,
+ session,
+ true,
+ new Flags());
+ }
}
diff --git a/third-party/rspamd/src/test/resources/json/feedHamClassifiedAsHam.additionalInformation.json b/third-party/rspamd/src/test/resources/json/feedHamClassifiedAsHam.additionalInformation.json
new file mode 100644
index 0000000000..11f189f3be
--- /dev/null
+++ b/third-party/rspamd/src/test/resources/json/feedHamClassifiedAsHam.additionalInformation.json
@@ -0,0 +1,12 @@
+{
+ "errorCount": 1,
+ "reportedHamMessageCount": 2,
+ "runningOptions": {
+ "messagesPerSecond": 10,
+ "samplingProbability": 1.0,
+ "classifiedAsSpam": false
+ },
+ "hamMessageCount": 4,
+ "timestamp": "2007-12-03T10:15:30Z",
+ "type": "FeedHamToRspamdTask"
+}
\ No newline at end of file
diff --git a/third-party/rspamd/src/test/resources/json/feedSpamClassifiedAsHam.additionalInformation.json b/third-party/rspamd/src/test/resources/json/feedSpamClassifiedAsHam.additionalInformation.json
new file mode 100644
index 0000000000..61260e5f27
--- /dev/null
+++ b/third-party/rspamd/src/test/resources/json/feedSpamClassifiedAsHam.additionalInformation.json
@@ -0,0 +1,12 @@
+{
+ "errorCount": 1,
+ "reportedSpamMessageCount": 2,
+ "runningOptions": {
+ "messagesPerSecond": 10,
+ "samplingProbability": 1.0,
+ "classifiedAsSpam": false
+ },
+ "spamMessageCount": 4,
+ "timestamp": "2007-12-03T10:15:30Z",
+ "type": "FeedSpamToRspamdTask"
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org