You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by ro...@apache.org on 2018/03/08 16:35:58 UTC
[08/23] james-project git commit: JAMES-2341 Correct SpamAssassin
learning
JAMES-2341 Correct SpamAssassin learning
Note that for now a static user is being used. This is changed later on.
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/697a5326
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/697a5326
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/697a5326
Branch: refs/heads/master
Commit: 697a5326ed17676c3b4f8e65deac4bb3026bc651
Parents: b92f3e9
Author: benwa <bt...@linagora.com>
Authored: Tue Mar 6 09:37:32 2018 +0700
Committer: Antoine Duprat <ad...@linagora.com>
Committed: Thu Mar 8 10:36:34 2018 +0100
----------------------------------------------------------------------
.../mailbox/spamassassin/SpamAssassin.java | 10 +++-
.../spamassassin/SpamAssassinListener.java | 6 +++
.../src/test/resources/spamassassin.properties | 2 +
server/container/guice/memory-guice/pom.xml | 4 ++
.../org/apache/james/MemoryJamesServerMain.java | 4 +-
.../modules/mailbox/MemoryMailboxModule.java | 2 +
.../src/test/resources/mailetcontainer.xml | 8 +++
.../src/test/resources/spamassassin.properties | 2 +
server/container/util/pom.xml | 4 ++
.../james/util/scanner/SpamAssassinInvoker.java | 19 +++++--
.../util/scanner/SpamAssassinExtension.java | 55 +++++++++-----------
.../util/scanner/SpamAssassinInvokerTest.java | 4 +-
12 files changed, 82 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassin.java
----------------------------------------------------------------------
diff --git a/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassin.java b/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassin.java
index 0dfb30e..864cba0 100644
--- a/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassin.java
+++ b/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassin.java
@@ -23,6 +23,11 @@ import java.util.List;
import javax.inject.Inject;
+import org.apache.james.util.Host;
+import org.apache.james.util.scanner.SpamAssassinInvoker;
+
+import com.github.fge.lambdas.Throwing;
+
public class SpamAssassin {
private final SpamAssassinConfiguration spamAssassinConfiguration;
@@ -34,7 +39,10 @@ public class SpamAssassin {
public void learnSpam(List<InputStream> messages) {
if (spamAssassinConfiguration.isEnable()) {
- // Will call SpamAssassinInvoker
+ Host host = spamAssassinConfiguration.getHost().get();
+ SpamAssassinInvoker invoker = new SpamAssassinInvoker(host.getHostName(), host.getPort());
+ messages
+ .forEach(Throwing.consumer(message -> invoker.learnAsSpam(message)));
}
}
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassinListener.java
----------------------------------------------------------------------
diff --git a/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassinListener.java b/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassinListener.java
index d0888f6..bccf22d 100644
--- a/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassinListener.java
+++ b/mailbox/plugin/spamassassin/src/main/java/org/apache/james/mailbox/spamassassin/SpamAssassinListener.java
@@ -26,6 +26,8 @@ import org.apache.james.mailbox.Role;
import org.apache.james.mailbox.store.event.EventFactory;
import org.apache.james.mailbox.store.event.SpamEventListener;
import org.apache.james.mailbox.store.mail.model.Message;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.github.fge.lambdas.Throwing;
import com.github.steveash.guavate.Guavate;
@@ -34,6 +36,8 @@ import com.google.common.collect.ImmutableList;
public class SpamAssassinListener implements SpamEventListener {
+ private static final Logger LOGGER = LoggerFactory.getLogger(SpamAssassinListener.class);
+
private final SpamAssassin spamAssassin;
@Inject
@@ -53,9 +57,11 @@ public class SpamAssassinListener implements SpamEventListener {
@Override
public void event(Event event) {
+ LOGGER.debug("Event {} received in listener.", event);
if (event instanceof EventFactory.AddedImpl) {
EventFactory.AddedImpl addedToMailboxEvent = (EventFactory.AddedImpl) event;
if (isEventOnSpamMailbox(addedToMailboxEvent)) {
+ LOGGER.debug("Spam event detected");
ImmutableList<InputStream> messages = addedToMailboxEvent.getAvailableMessages()
.values()
.stream()
http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/guice/cassandra-guice/src/test/resources/spamassassin.properties
----------------------------------------------------------------------
diff --git a/server/container/guice/cassandra-guice/src/test/resources/spamassassin.properties b/server/container/guice/cassandra-guice/src/test/resources/spamassassin.properties
new file mode 100644
index 0000000..134ed31
--- /dev/null
+++ b/server/container/guice/cassandra-guice/src/test/resources/spamassassin.properties
@@ -0,0 +1,2 @@
+spamassassin.host=localhost
+spamassassin.host=783
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/guice/memory-guice/pom.xml
----------------------------------------------------------------------
diff --git a/server/container/guice/memory-guice/pom.xml b/server/container/guice/memory-guice/pom.xml
index b3f4c83..b839cfb 100644
--- a/server/container/guice/memory-guice/pom.xml
+++ b/server/container/guice/memory-guice/pom.xml
@@ -88,6 +88,10 @@
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
+ <artifactId>james-server-guice-mailbox-plugin-spamassassin</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
<artifactId>james-server-guice-managedsieve</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/guice/memory-guice/src/main/java/org/apache/james/MemoryJamesServerMain.java
----------------------------------------------------------------------
diff --git a/server/container/guice/memory-guice/src/main/java/org/apache/james/MemoryJamesServerMain.java b/server/container/guice/memory-guice/src/main/java/org/apache/james/MemoryJamesServerMain.java
index 3170cf4..b7b3a1b 100644
--- a/server/container/guice/memory-guice/src/main/java/org/apache/james/MemoryJamesServerMain.java
+++ b/server/container/guice/memory-guice/src/main/java/org/apache/james/MemoryJamesServerMain.java
@@ -41,6 +41,7 @@ import org.apache.james.modules.server.MemoryMailQueueModule;
import org.apache.james.modules.server.RawPostDequeueDecoratorModule;
import org.apache.james.modules.server.SwaggerRoutesModule;
import org.apache.james.modules.server.WebAdminServerModule;
+import org.apache.james.modules.spamassassin.SpamAssassinListenerModule;
import com.google.inject.Module;
import com.google.inject.util.Modules;
@@ -61,7 +62,8 @@ public class MemoryJamesServerMain {
new ManageSieveServerModule(),
new POP3ServerModule(),
new ProtocolHandlerModule(),
- new SMTPServerModule());
+ new SMTPServerModule(),
+ new SpamAssassinListenerModule());
public static final Module JMAP = Modules.combine(
new MemoryDataJmapModule(),
http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/mailbox/MemoryMailboxModule.java
----------------------------------------------------------------------
diff --git a/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/mailbox/MemoryMailboxModule.java b/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/mailbox/MemoryMailboxModule.java
index de9b544..34041f5 100644
--- a/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/mailbox/MemoryMailboxModule.java
+++ b/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/mailbox/MemoryMailboxModule.java
@@ -49,6 +49,7 @@ import org.apache.james.mailbox.store.JVMMailboxPathLocker;
import org.apache.james.mailbox.store.MailboxSessionMapperFactory;
import org.apache.james.mailbox.store.StoreAttachmentManager;
import org.apache.james.mailbox.store.StoreBlobManager;
+import org.apache.james.mailbox.store.StoreMailboxManager;
import org.apache.james.mailbox.store.StoreMessageIdManager;
import org.apache.james.mailbox.store.StoreRightManager;
import org.apache.james.mailbox.store.StoreSubscriptionManager;
@@ -96,6 +97,7 @@ public class MemoryMailboxModule extends AbstractModule {
bind(Authenticator.class).to(UserRepositoryAuthenticator.class);
bind(Authorizator.class).to(UserRepositoryAuthorizator.class);
bind(MailboxManager.class).to(InMemoryMailboxManager.class);
+ bind(StoreMailboxManager.class).to(InMemoryMailboxManager.class);
bind(MessageIdManager.class).to(StoreMessageIdManager.class);
bind(AttachmentManager.class).to(StoreAttachmentManager.class);
http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/guice/memory-guice/src/test/resources/mailetcontainer.xml
----------------------------------------------------------------------
diff --git a/server/container/guice/memory-guice/src/test/resources/mailetcontainer.xml b/server/container/guice/memory-guice/src/test/resources/mailetcontainer.xml
index 887dab3..71de139 100644
--- a/server/container/guice/memory-guice/src/test/resources/mailetcontainer.xml
+++ b/server/container/guice/memory-guice/src/test/resources/mailetcontainer.xml
@@ -56,6 +56,14 @@
</mailet>
<mailet match="All" class="RecipientRewriteTable" />
<mailet match="RecipientIsLocal" class="org.apache.james.jmap.mailet.VacationMailet"/>
+ <mailet match="RecipientIsLocal" class="SpamAssassin">
+ <spamdHost>localhost</spamdHost>
+ <spamdPort>783</spamdPort>
+ </mailet>
+ <mailet match="IsMarkedAsSpam" class="ToRecipientFolder">
+ <folder>Spam</folder>
+ <consume>true</consume>
+ </mailet>
<mailet match="RecipientIsLocal" class="LocalDelivery"/>
<mailet match="HostIsLocal" class="ToProcessor">
<processor>local-address-error</processor>
http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/guice/memory-guice/src/test/resources/spamassassin.properties
----------------------------------------------------------------------
diff --git a/server/container/guice/memory-guice/src/test/resources/spamassassin.properties b/server/container/guice/memory-guice/src/test/resources/spamassassin.properties
new file mode 100644
index 0000000..134ed31
--- /dev/null
+++ b/server/container/guice/memory-guice/src/test/resources/spamassassin.properties
@@ -0,0 +1,2 @@
+spamassassin.host=localhost
+spamassassin.host=783
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/util/pom.xml
----------------------------------------------------------------------
diff --git a/server/container/util/pom.xml b/server/container/util/pom.xml
index 1128afd..c602027 100644
--- a/server/container/util/pom.xml
+++ b/server/container/util/pom.xml
@@ -52,6 +52,10 @@
<artifactId>javax.mail</artifactId>
</dependency>
<dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/util/src/main/java/org/apache/james/util/scanner/SpamAssassinInvoker.java
----------------------------------------------------------------------
diff --git a/server/container/util/src/main/java/org/apache/james/util/scanner/SpamAssassinInvoker.java b/server/container/util/src/main/java/org/apache/james/util/scanner/SpamAssassinInvoker.java
index 95063da..3e6f856 100644
--- a/server/container/util/src/main/java/org/apache/james/util/scanner/SpamAssassinInvoker.java
+++ b/server/container/util/src/main/java/org/apache/james/util/scanner/SpamAssassinInvoker.java
@@ -33,6 +33,8 @@ import javax.mail.MessagingException;
import javax.mail.internet.MimeMessage;
import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
@@ -42,6 +44,7 @@ import com.google.common.collect.Lists;
* href="SpamAssassin.org">SpamAssassin.org</a> for info on configuration.
*/
public class SpamAssassinInvoker {
+ private static final Logger LOGGER = LoggerFactory.getLogger(SpamAssassinInvoker.class);
/** The mail attribute under which the status get stored */
public static final String STATUS_MAIL_ATTRIBUTE_NAME = "org.apache.james.spamassassin.status";
@@ -88,6 +91,8 @@ public class SpamAssassinInvoker {
writer.write("CHECK SPAMC/1.2");
writer.write(CRLF);
+ writer.write("User: alice@angels.org");
+ writer.write(CRLF);
writer.write(CRLF);
writer.flush();
@@ -131,6 +136,7 @@ public class SpamAssassinInvoker {
try {
return Boolean.valueOf(string);
} catch (Exception e) {
+ LOGGER.warn("Fail parsing spamassassin answer: " + string);
return false;
}
}
@@ -153,8 +159,13 @@ public class SpamAssassinInvoker {
PrintWriter writer = new PrintWriter(out);
BufferedReader in = new BufferedReader(new InputStreamReader(socket.getInputStream()))) {
+ byte[] byteArray = IOUtils.toByteArray(message);
writer.write("TELL SPAMC/1.2");
writer.write(CRLF);
+ writer.write("Content-length: " + byteArray.length);
+ writer.write(CRLF);
+ writer.write("User: alice@angels.org");
+ writer.write(CRLF);
writer.write("Message-class: spam");
writer.write(CRLF);
writer.write("Set: local, remote");
@@ -162,14 +173,12 @@ public class SpamAssassinInvoker {
writer.write(CRLF);
writer.flush();
- IOUtils.copy(message, out);
+ out.write(byteArray);
out.flush();
socket.shutdownOutput();
return in.lines()
- .filter(this::hasBeenSet)
- .findAny()
- .isPresent();
+ .anyMatch(this::hasBeenSet);
} catch (UnknownHostException e) {
throw new MessagingException("Error communicating with spamd. Unknown host: " + spamdHost);
} catch (IOException e) {
@@ -178,6 +187,6 @@ public class SpamAssassinInvoker {
}
private boolean hasBeenSet(String line) {
- return line.startsWith("DidSet");
+ return line.startsWith("DidSet: local");
}
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinExtension.java
----------------------------------------------------------------------
diff --git a/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinExtension.java b/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinExtension.java
index c185be6..e161804 100644
--- a/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinExtension.java
+++ b/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinExtension.java
@@ -19,19 +19,14 @@
package org.apache.james.util.scanner;
-import java.io.FileInputStream;
import java.io.IOException;
-import java.io.InputStream;
import java.net.URISyntaxException;
-import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.function.Function;
+import java.util.Locale;
import java.util.stream.Stream;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.tuple.Pair;
import org.junit.jupiter.api.extension.AfterAllCallback;
import org.junit.jupiter.api.extension.BeforeAllCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
@@ -72,7 +67,11 @@ public class SpamAssassinExtension implements BeforeAllCallback, AfterAllCallbac
public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException {
return spamAssassin;
}
-
+
+ public SpamAssassin getSpamAssassin() {
+ return spamAssassin;
+ }
+
public static class SpamAssassin {
private static final int SPAMASSASSIN_PORT = 783;
@@ -95,38 +94,28 @@ public class SpamAssassinExtension implements BeforeAllCallback, AfterAllCallbac
return bindingPort;
}
- public void train() throws IOException, URISyntaxException {
- train(Paths.get(ClassLoader.getSystemResource("spamassassin_db/spam").toURI()), TrainingKind.SPAM);
- train(Paths.get(ClassLoader.getSystemResource("spamassassin_db/ham").toURI()), TrainingKind.HAM);
+ public void train(String user) throws IOException, URISyntaxException {
+ train(user, Paths.get(ClassLoader.getSystemResource("spamassassin_db/spam").toURI()), TrainingKind.SPAM);
+ train(user, Paths.get(ClassLoader.getSystemResource("spamassassin_db/ham").toURI()), TrainingKind.HAM);
}
- private void train(Path folder, TrainingKind trainingKind) throws URISyntaxException, IOException {
+ private void train(String user, Path folder, TrainingKind trainingKind) throws URISyntaxException, IOException {
+ spamAssassinContainer.getDockerClient().copyArchiveToContainerCmd(spamAssassinContainer.getContainerId())
+ .withHostResource(folder.toAbsolutePath().toString())
+ .withRemotePath("/root")
+ .exec();
try (Stream<Path> paths = Files.walk(folder)) {
paths
.filter(Files::isRegularFile)
.map(Path::toFile)
- .map(Throwing.function(FileInputStream::new))
- .map(readAsPair())
- .map(closeStream())
- .map(Pair::getRight)
- .forEach(Throwing.consumer(message ->
+ .forEach(Throwing.consumer(file -> {
spamAssassinContainer.execInContainer("sa-learn",
- trainingKind.saLearnExtensionName(),
- message)));
+ trainingKind.saLearnExtensionName(), "-u", user,
+ "/root/" + trainingKind.name().toLowerCase(Locale.US) + "/" + file.getName());
+ }));
}
}
- private Function<InputStream, Pair<InputStream, String>> readAsPair() {
- return Throwing.function(inputStream -> Pair.of(inputStream, IOUtils.toString(inputStream, StandardCharsets.UTF_8)));
- }
-
- private Function<Pair<InputStream, String>, Pair<InputStream, String>> closeStream() {
- return Throwing.function(pair -> {
- pair.getLeft().close();
- return pair;
- });
- }
-
private static enum TrainingKind {
SPAM("--spam"), HAM("--ham");
@@ -140,6 +129,14 @@ public class SpamAssassinExtension implements BeforeAllCallback, AfterAllCallbac
return saLearnExtensionName;
}
}
+
+ public void sync(String user) throws UnsupportedOperationException, IOException, InterruptedException {
+ spamAssassinContainer.execInContainer("sa-learn", "--sync", "-u", user);
+ }
+
+ public void dump(String user) throws UnsupportedOperationException, IOException, InterruptedException {
+ spamAssassinContainer.execInContainer("sa-learn", "--dump", "magic", "-u", user);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/697a5326/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinInvokerTest.java
----------------------------------------------------------------------
diff --git a/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinInvokerTest.java b/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinInvokerTest.java
index fdf3b83..ffa796a 100644
--- a/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinInvokerTest.java
+++ b/server/container/util/src/test/java/org/apache/james/util/scanner/SpamAssassinInvokerTest.java
@@ -70,7 +70,7 @@ public class SpamAssassinInvokerTest {
@Test
public void scanMailShouldMarkHasSpamWhenKnownHasSpam() throws Exception {
- spamAssassin.train();
+ spamAssassin.train("user");
MimeMessage mimeMessage = MimeMessageUtil.mimeMessageFromStream(
ClassLoader.getSystemResourceAsStream("spamassassin_db/spam/spam1"));
@@ -91,7 +91,7 @@ public class SpamAssassinInvokerTest {
}
@Test
- public void scanMailShouldMarkHasSpamWhenMessageAlreadyLearnedAsSpam() throws Exception {
+ public void scanMailShouldMarkAsSpamWhenMessageAlreadyLearnedAsSpam() throws Exception {
MimeMessage mimeMessage = MimeMessageUtil.mimeMessageFromStream(
ClassLoader.getSystemResourceAsStream("spamassassin_db/spam/spam1"));
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org