You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2018/06/19 09:53:52 UTC
[6/8] james-project git commit: JAMES-2429 Implement basic Dlp matcher
JAMES-2429 Implement basic Dlp matcher
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/a8586df3
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/a8586df3
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/a8586df3
Branch: refs/heads/master
Commit: a8586df34d82dd50e9ea387e6b0becdc34fbae66
Parents: 19caa77
Author: Matthieu Baechler <ma...@apache.org>
Authored: Wed Jun 6 08:55:24 2018 +0200
Committer: benwa <bt...@linagora.com>
Committed: Tue Jun 19 16:53:02 2018 +0700
----------------------------------------------------------------------
.../apache/mailet/base/MailAddressFixture.java | 5 +
.../james/dlp/api/DLPConfigurationLoader.java | 28 +
.../james/dlp/api/DLPConfigurationStore.java | 5 +-
.../james/transport/matchers/dlp/Dlp.java | 78 +++
.../transport/matchers/dlp/DlpDomainRules.java | 300 +++++++++++
.../transport/matchers/dlp/DlpRulesLoader.java | 57 ++
.../matchers/dlp/DlpDomainRulesTest.java | 60 +++
.../james/transport/matchers/dlp/DlpTest.java | 514 +++++++++++++++++++
8 files changed, 1043 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/a8586df3/mailet/base/src/test/java/org/apache/mailet/base/MailAddressFixture.java
----------------------------------------------------------------------
diff --git a/mailet/base/src/test/java/org/apache/mailet/base/MailAddressFixture.java b/mailet/base/src/test/java/org/apache/mailet/base/MailAddressFixture.java
index 51bd487..4efe093 100644
--- a/mailet/base/src/test/java/org/apache/mailet/base/MailAddressFixture.java
+++ b/mailet/base/src/test/java/org/apache/mailet/base/MailAddressFixture.java
@@ -21,6 +21,7 @@ package org.apache.mailet.base;
import javax.mail.internet.AddressException;
+import org.apache.james.core.Domain;
import org.apache.james.core.MailAddress;
public class MailAddressFixture {
@@ -28,6 +29,10 @@ public class MailAddressFixture {
public static final String JAMES_APACHE_ORG = "james.apache.org";
public static final String JAMES2_APACHE_ORG = "james2.apache.org";
+ public static final Domain JAMES_LOCAL_DOMAIN = Domain.of(JAMES_LOCAL);
+ public static final Domain JAMES_APACHE_ORG_DOMAIN = Domain.of(JAMES_APACHE_ORG);
+ public static final Domain JAMES2_APACHE_ORG_DOMAIN = Domain.of(JAMES2_APACHE_ORG);
+
public static final MailAddress SENDER = createMailAddress("sender@" + JAMES_LOCAL);
public static final MailAddress RECIPIENT1 = createMailAddress("recipient1@" + JAMES_LOCAL);
public static final MailAddress RECIPIENT2 = createMailAddress("recipient2@" + JAMES_LOCAL);
http://git-wip-us.apache.org/repos/asf/james-project/blob/a8586df3/server/data/data-api/src/main/java/org/apache/james/dlp/api/DLPConfigurationLoader.java
----------------------------------------------------------------------
diff --git a/server/data/data-api/src/main/java/org/apache/james/dlp/api/DLPConfigurationLoader.java b/server/data/data-api/src/main/java/org/apache/james/dlp/api/DLPConfigurationLoader.java
new file mode 100644
index 0000000..e2d27f5
--- /dev/null
+++ b/server/data/data-api/src/main/java/org/apache/james/dlp/api/DLPConfigurationLoader.java
@@ -0,0 +1,28 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.dlp.api;
+
+import java.util.stream.Stream;
+
+import org.apache.james.core.Domain;
+
+public interface DLPConfigurationLoader {
+ Stream<DLPConfigurationItem> list(Domain domain);
+}
http://git-wip-us.apache.org/repos/asf/james-project/blob/a8586df3/server/data/data-api/src/main/java/org/apache/james/dlp/api/DLPConfigurationStore.java
----------------------------------------------------------------------
diff --git a/server/data/data-api/src/main/java/org/apache/james/dlp/api/DLPConfigurationStore.java b/server/data/data-api/src/main/java/org/apache/james/dlp/api/DLPConfigurationStore.java
index 457341e..4272f49 100644
--- a/server/data/data-api/src/main/java/org/apache/james/dlp/api/DLPConfigurationStore.java
+++ b/server/data/data-api/src/main/java/org/apache/james/dlp/api/DLPConfigurationStore.java
@@ -20,15 +20,12 @@
package org.apache.james.dlp.api;
import java.util.List;
-import java.util.stream.Stream;
import org.apache.james.core.Domain;
import com.google.common.collect.ImmutableList;
-public interface DLPConfigurationStore {
-
- Stream<DLPConfigurationItem> list(Domain domain);
+public interface DLPConfigurationStore extends DLPConfigurationLoader {
void store(Domain domain, List<DLPConfigurationItem> rule);
http://git-wip-us.apache.org/repos/asf/james-project/blob/a8586df3/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/Dlp.java
----------------------------------------------------------------------
diff --git a/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/Dlp.java b/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/Dlp.java
new file mode 100644
index 0000000..cb7fc72
--- /dev/null
+++ b/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/Dlp.java
@@ -0,0 +1,78 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.transport.matchers.dlp;
+
+import java.util.Collection;
+import java.util.Optional;
+
+import javax.inject.Inject;
+import javax.mail.MessagingException;
+
+import org.apache.james.core.MailAddress;
+import org.apache.james.dlp.api.DLPConfigurationItem;
+import org.apache.mailet.Mail;
+import org.apache.mailet.base.GenericMatcher;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+
+public class Dlp extends GenericMatcher {
+
+ public static final String DLP_MATCHED_RULE = "DlpMatchedRule";
+
+ private final DlpRulesLoader rulesLoader;
+
+ @Inject
+ @VisibleForTesting
+ Dlp(DlpRulesLoader rulesLoader) {
+ this.rulesLoader = rulesLoader;
+ }
+
+ @Override
+ public Collection<MailAddress> match(Mail mail) throws MessagingException {
+ Optional<DLPConfigurationItem.Id> firstMatchingRuleId = findFirstMatchingRule(mail);
+
+ if (firstMatchingRuleId.isPresent()) {
+ DLPConfigurationItem.Id ruleId = firstMatchingRuleId.get();
+ setRuleIdAsMailAttribute(mail, ruleId);
+ return mail.getRecipients();
+ }
+ return ImmutableList.of();
+ }
+
+ private void setRuleIdAsMailAttribute(Mail mail, DLPConfigurationItem.Id ruleId) {
+ mail.setAttribute(DLP_MATCHED_RULE, ruleId.asString());
+ }
+
+ private Optional<DLPConfigurationItem.Id> findFirstMatchingRule(Mail mail) {
+ return Optional
+ .ofNullable(mail.getSender())
+ .flatMap(sender -> matchingRule(sender, mail));
+ }
+
+ private Optional<DLPConfigurationItem.Id> matchingRule(MailAddress address, Mail mail) {
+ return rulesLoader.load(address.getDomain()).match(mail);
+ }
+
+ @Override
+ public String getMatcherInfo() {
+ return "Data Leak Prevention Matcher";
+ }
+}
http://git-wip-us.apache.org/repos/asf/james-project/blob/a8586df3/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpDomainRules.java
----------------------------------------------------------------------
diff --git a/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpDomainRules.java b/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpDomainRules.java
new file mode 100644
index 0000000..f44a60e
--- /dev/null
+++ b/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpDomainRules.java
@@ -0,0 +1,300 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.transport.matchers.dlp;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
+import javax.mail.Address;
+import javax.mail.BodyPart;
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import javax.mail.Multipart;
+import javax.mail.internet.MimeMessage;
+
+import org.apache.james.core.MailAddress;
+import org.apache.james.dlp.api.DLPConfigurationItem;
+import org.apache.james.dlp.api.DLPConfigurationItem.Targets;
+import org.apache.james.javax.MultipartUtil;
+import org.apache.james.mime4j.util.MimeUtil;
+import org.apache.james.util.OptionalUtils;
+import org.apache.mailet.Mail;
+
+import com.github.fge.lambdas.Throwing;
+import com.github.fge.lambdas.predicates.ThrowingPredicate;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMultimap;
+
+public class DlpDomainRules {
+
+ @VisibleForTesting static DlpDomainRules matchNothing() {
+ return DlpDomainRules.of(new Rule(DLPConfigurationItem.Id.of("always false"), (mail) -> false));
+ }
+
+ @VisibleForTesting static DlpDomainRules matchAll() {
+ return DlpDomainRules.of(new Rule(DLPConfigurationItem.Id.of("always true"), (mail) -> true));
+ }
+
+ private static DlpDomainRules of(Rule rule) {
+ return new DlpDomainRules(ImmutableList.of(rule));
+ }
+
+ public static DlpDomainRulesBuilder builder() {
+ return new DlpDomainRulesBuilder();
+ }
+
+ static class Rule {
+
+ interface MatcherFunction extends ThrowingPredicate<Mail> { }
+
+
+ private static Stream<String> asStringStream(Address[] addresses) {
+ return Arrays.stream(addresses).map(Rule::asString);
+ }
+
+ private static String asString(Address address) {
+ return MimeUtil.unscrambleHeaderValue(address.toString());
+ }
+
+ private static class ContentMatcher implements Rule.MatcherFunction {
+
+ private final Pattern pattern;
+
+ private ContentMatcher(Pattern pattern) {
+ this.pattern = pattern;
+ }
+
+ @Override
+ public boolean doTest(Mail mail) throws MessagingException, IOException {
+ return Stream
+ .concat(getMessageSubjects(mail), getMessageBodies(mail.getMessage()))
+ .anyMatch(pattern.asPredicate());
+ }
+
+ private Stream<String> getMessageSubjects(Mail mail) throws MessagingException {
+ MimeMessage message = mail.getMessage();
+ if (message != null) {
+ return OptionalUtils.toStream(
+ Optional.ofNullable(message.getSubject()));
+ }
+ return Stream.of();
+ }
+
+ private Stream<String> getMessageBodies(Message message) throws MessagingException, IOException {
+ if (message != null) {
+ return getMessageBodiesFromContent(message.getContent());
+ }
+ return Stream.of();
+ }
+
+ private Stream<String> getMessageBodiesFromContent(Object content) throws IOException, MessagingException {
+ if (content instanceof String) {
+ return Stream.of((String) content);
+ }
+ if (content instanceof Message) {
+ Message message = (Message) content;
+ return getMessageBodiesFromContent(message.getContent());
+ }
+ if (content instanceof Multipart) {
+ return MultipartUtil.retrieveBodyParts((Multipart) content)
+ .stream()
+ .map(Throwing.function(BodyPart::getContent).sneakyThrow())
+ .flatMap(Throwing.function(this::getMessageBodiesFromContent).sneakyThrow());
+ }
+ return Stream.of();
+ }
+ }
+
+ private static class RecipientsMatcher implements Rule.MatcherFunction {
+
+ private final Pattern pattern;
+
+ private RecipientsMatcher(Pattern pattern) {
+ this.pattern = pattern;
+ }
+
+ @Override
+ public boolean doTest(Mail mail) throws MessagingException, IOException {
+ return listRecipientsAsString(mail).anyMatch(pattern.asPredicate());
+ }
+
+ private Stream<String> listRecipientsAsString(Mail mail) throws MessagingException {
+ return Stream.concat(listEnvelopRecipients(mail), listHeaderRecipients(mail));
+ }
+
+ private Stream<String> listEnvelopRecipients(Mail mail) {
+ return mail.getRecipients().stream().map(MailAddress::asString);
+ }
+
+ private Stream<String> listHeaderRecipients(Mail mail) throws MessagingException {
+ return Optional.ofNullable(mail.getMessage())
+ .flatMap(Throwing.function(m -> Optional.ofNullable(m.getAllRecipients())))
+ .map(Rule::asStringStream)
+ .orElse(Stream.of());
+ }
+
+ }
+
+ private static class SenderMatcher implements Rule.MatcherFunction {
+
+ private final Pattern pattern;
+
+ private SenderMatcher(Pattern pattern) {
+ this.pattern = pattern;
+ }
+
+ @Override
+ public boolean doTest(Mail mail) throws MessagingException {
+ return listSenders(mail).anyMatch(pattern.asPredicate());
+ }
+
+ private Stream<String> listSenders(Mail mail) throws MessagingException {
+ return Stream.concat(listEnvelopSender(mail), listFromHeaders(mail));
+ }
+
+ private Stream<String> listEnvelopSender(Mail mail) {
+ return OptionalUtils.toStream(Optional.ofNullable(mail.getSender()).map(MailAddress::asString));
+ }
+
+ private Stream<String> listFromHeaders(Mail mail) throws MessagingException {
+ MimeMessage message = mail.getMessage();
+ if (message != null) {
+ return asStringStream(message.getFrom());
+ }
+ return Stream.of();
+ }
+
+ }
+
+ private final DLPConfigurationItem.Id id;
+ private final MatcherFunction matcher;
+
+ public Rule(DLPConfigurationItem.Id id, MatcherFunction matcher) {
+ this.id = id;
+ this.matcher = matcher;
+ }
+
+ public DLPConfigurationItem.Id id() {
+ return id;
+ }
+
+ public boolean match(Mail mail) {
+ return matcher.test(mail);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof Rule) {
+ Rule other = (Rule) o;
+ return Objects.equals(id, other.id) &&
+ Objects.equals(matcher, other.matcher);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id, matcher);
+ }
+
+ }
+
+ public static class DlpDomainRulesBuilder {
+
+ private final ImmutableMultimap.Builder<Targets.Type, Rule> rules;
+
+ private DlpDomainRulesBuilder() {
+ rules = ImmutableMultimap.builder();
+ }
+
+ public DlpDomainRulesBuilder recipientRule(DLPConfigurationItem.Id id, Pattern pattern) {
+ return rule(Targets.Type.Recipient, id, pattern);
+ }
+
+ public DlpDomainRulesBuilder senderRule(DLPConfigurationItem.Id id, Pattern pattern) {
+ return rule(Targets.Type.Sender, id, pattern);
+ }
+
+ public DlpDomainRulesBuilder contentRule(DLPConfigurationItem.Id id, Pattern pattern) {
+ return rule(Targets.Type.Content, id, pattern);
+ }
+
+ public DlpDomainRulesBuilder rule(Targets.Type type, DLPConfigurationItem.Id id, Pattern regexp) {
+ rules.put(type, toRule(type, id, regexp));
+ return this;
+ }
+
+ private Rule toRule(Targets.Type type, DLPConfigurationItem.Id id, Pattern pattern) {
+ switch (type) {
+ case Sender:
+ return new Rule(id, new Rule.SenderMatcher(pattern));
+ case Content:
+ return new Rule(id, new Rule.ContentMatcher(pattern));
+ case Recipient:
+ return new Rule(id, new Rule.RecipientsMatcher(pattern));
+ default:
+ throw new IllegalArgumentException("unexpected value");
+ }
+ }
+
+ public DlpDomainRules build() {
+ ImmutableMultimap<Targets.Type, Rule> rules = this.rules.build();
+ Preconditions.checkState(!containsDuplicateIds(rules), "Rules should not contain duplicated `id`");
+ return new DlpDomainRules(rules.values());
+ }
+
+ private boolean containsDuplicateIds(ImmutableMultimap<Targets.Type, Rule> rules) {
+ return
+ Stream.of(Targets.Type.values())
+ .map(rules::get)
+ .anyMatch(this::containsDuplicateIds);
+ }
+
+ private boolean containsDuplicateIds(ImmutableCollection<Rule> rules) {
+ long distinctIdCount = rules.stream()
+ .map(Rule::id)
+ .distinct()
+ .count();
+ return distinctIdCount != rules.size();
+ }
+
+ }
+
+ private final ImmutableCollection<Rule> rules;
+
+ private DlpDomainRules(ImmutableCollection<Rule> rules) {
+ this.rules = rules;
+ }
+
+ public Optional<DLPConfigurationItem.Id> match(Mail mail) {
+ return rules.stream()
+ .filter(rule -> rule.match(mail))
+ .map(Rule::id)
+ .findFirst();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/james-project/blob/a8586df3/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpRulesLoader.java
----------------------------------------------------------------------
diff --git a/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpRulesLoader.java b/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpRulesLoader.java
new file mode 100644
index 0000000..13430d9
--- /dev/null
+++ b/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpRulesLoader.java
@@ -0,0 +1,57 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.transport.matchers.dlp;
+
+import java.util.stream.Stream;
+
+import javax.inject.Inject;
+
+import org.apache.james.core.Domain;
+import org.apache.james.dlp.api.DLPConfigurationItem;
+import org.apache.james.dlp.api.DLPConfigurationStore;
+
+public interface DlpRulesLoader {
+
+ DlpDomainRules load(Domain domain);
+
+ class Impl implements DlpRulesLoader {
+
+ private final DLPConfigurationStore configurationStore;
+
+ @Inject
+ public Impl(DLPConfigurationStore configurationStore) {
+ this.configurationStore = configurationStore;
+ }
+
+ @Override
+ public DlpDomainRules load(Domain domain) {
+ return toRules(configurationStore.list(domain));
+ }
+
+ private DlpDomainRules toRules(Stream<DLPConfigurationItem> items) {
+ DlpDomainRules.DlpDomainRulesBuilder builder = DlpDomainRules.builder();
+ items.forEach(item ->
+ item.getTargets().list().forEach(type ->
+ builder.rule(type, item.getId(), item.getRegexp())
+ ));
+ return builder.build();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/james-project/blob/a8586df3/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpDomainRulesTest.java
----------------------------------------------------------------------
diff --git a/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpDomainRulesTest.java b/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpDomainRulesTest.java
new file mode 100644
index 0000000..155795f
--- /dev/null
+++ b/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpDomainRulesTest.java
@@ -0,0 +1,60 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.transport.matchers.dlp;
+
+import static org.assertj.core.api.Assertions.assertThatCode;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.util.regex.Pattern;
+
+import org.apache.james.dlp.api.DLPConfigurationItem.Id;
+import org.junit.jupiter.api.Test;
+
+class DlpDomainRulesTest {
+
+ private static final Pattern PATTERN_1 = Pattern.compile("1");
+ private static final Pattern PATTERN_2 = Pattern.compile("2");
+
+ @Test
+ void builderShouldThrowWhenDuplicateIds() {
+ assertThatThrownBy(() -> DlpDomainRules.builder()
+ .senderRule(Id.of("1"), PATTERN_1)
+ .senderRule(Id.of("1"), PATTERN_2)
+ .build())
+ .isInstanceOf(IllegalStateException.class);
+ }
+
+ @Test
+ void builderShouldNotThrowWhenDuplicateIdsOnDifferentTypes() {
+ assertThatCode(() -> DlpDomainRules.builder()
+ .senderRule(Id.of("1"), PATTERN_1)
+ .contentRule(Id.of("1"), PATTERN_2)
+ .build())
+ .doesNotThrowAnyException();
+ }
+
+
+ @Test
+ void builderShouldNotThrowWhenEmpty() {
+ assertThatCode(() -> DlpDomainRules.builder().build())
+ .doesNotThrowAnyException();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/james-project/blob/a8586df3/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpTest.java
----------------------------------------------------------------------
diff --git a/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpTest.java b/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpTest.java
new file mode 100644
index 0000000..12aa394
--- /dev/null
+++ b/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpTest.java
@@ -0,0 +1,514 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.transport.matchers.dlp;
+
+import static org.apache.mailet.base.MailAddressFixture.ANY_AT_JAMES;
+import static org.apache.mailet.base.MailAddressFixture.JAMES_APACHE_ORG;
+import static org.apache.mailet.base.MailAddressFixture.JAMES_APACHE_ORG_DOMAIN;
+import static org.apache.mailet.base.MailAddressFixture.OTHER_AT_JAMES;
+import static org.apache.mailet.base.MailAddressFixture.RECIPIENT1;
+import static org.apache.mailet.base.MailAddressFixture.RECIPIENT2;
+import static org.apache.mailet.base.MailAddressFixture.RECIPIENT3;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import java.util.regex.Pattern;
+
+import org.apache.james.core.Domain;
+import org.apache.james.core.builder.MimeMessageBuilder;
+import org.apache.james.dlp.api.DLPConfigurationItem.Id;
+import org.apache.mailet.base.test.FakeMail;
+import org.junit.jupiter.api.Test;
+
+class DlpTest {
+
+ private static final DlpRulesLoader MATCH_ALL_FOR_ALL_DOMAINS = (Domain domain) -> DlpDomainRules.matchAll();
+ private static final DlpRulesLoader MATCH_NOTHING_FOR_ALL_DOMAINS = (Domain domain) -> DlpDomainRules.matchNothing();
+
+ private static DlpRulesLoader asRulesLoaderFor(Domain domain, DlpDomainRules rules) {
+ return (Domain d) -> Optional
+ .of(d)
+ .filter(domain::equals)
+ .map(ignore -> rules)
+ .orElse(DlpDomainRules.matchNothing());
+ }
+
+ @Test
+ void matchShouldReturnEmptyWhenNoRecipient() throws Exception {
+ Dlp dlp = new Dlp(MATCH_ALL_FOR_ALL_DOMAINS);
+
+ FakeMail mail = FakeMail.builder().sender(RECIPIENT1).build();
+
+ assertThat(dlp.match(mail)).isEmpty();
+ }
+
+ @Test
+ void matchShouldReturnEmptyWhenNoSender() throws Exception {
+ Dlp dlp = new Dlp(MATCH_ALL_FOR_ALL_DOMAINS);
+
+ FakeMail mail = FakeMail.builder().recipient(RECIPIENT1).build();
+
+ assertThat(dlp.match(mail)).isEmpty();
+ }
+
+ @Test
+ void matchShouldThrowOnNullMail() {
+ Dlp dlp = new Dlp(MATCH_ALL_FOR_ALL_DOMAINS);
+
+ assertThatThrownBy(() -> dlp.match(null)).isInstanceOf(NullPointerException.class);
+ }
+
+ @Test
+ void matchShouldReturnEmptyWhenNoRuleMatch() throws Exception {
+ Dlp dlp = new Dlp(MATCH_NOTHING_FOR_ALL_DOMAINS);
+
+ FakeMail mail = FakeMail.builder()
+ .sender(ANY_AT_JAMES)
+ .recipient(RECIPIENT1)
+ .recipient(RECIPIENT2)
+ .build();
+
+ assertThat(dlp.match(mail)).isEmpty();
+ }
+
+ @Test
+ void matchSenderShouldReturnRecipientsWhenEnvelopSenderMatches() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder().senderRule(Id.of("match sender"), Pattern.compile(ANY_AT_JAMES.asString())).build()));
+
+ FakeMail mail = FakeMail.builder().sender(ANY_AT_JAMES).recipient(RECIPIENT1).build();
+
+ assertThat(dlp.match(mail)).contains(RECIPIENT1);
+ }
+
+ @Test
+ void matchSenderShouldReturnRecipientsWhenFromHeaderMatches() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder().senderRule(Id.of("match sender"), Pattern.compile(ANY_AT_JAMES.asString())).build()));
+
+ FakeMail mail = FakeMail
+ .builder()
+ .sender(OTHER_AT_JAMES)
+ .recipient(RECIPIENT1)
+ .mimeMessage(MimeMessageBuilder
+ .mimeMessageBuilder()
+ .addFrom(ANY_AT_JAMES.toInternetAddress()))
+ .build();
+
+ assertThat(dlp.match(mail)).contains(RECIPIENT1);
+ }
+
+ @Test
+ void matchShouldReturnRecipientsWhenEnvelopRecipientsMatches() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder().recipientRule(Id.of("match recipient"), Pattern.compile(RECIPIENT1.asString())).build()));
+
+ FakeMail mail = FakeMail.builder()
+ .sender(ANY_AT_JAMES)
+ .recipient(RECIPIENT1)
+ .recipient(RECIPIENT2)
+ .build();
+
+ assertThat(dlp.match(mail)).contains(RECIPIENT1, RECIPIENT2);
+ }
+
+ @Test
+ void matchShouldReturnRecipientsWhenToHeaderMatches() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder().recipientRule(Id.of("match recipient"), Pattern.compile(RECIPIENT2.asString())).build()));
+
+ FakeMail mail = FakeMail
+ .builder()
+ .sender(OTHER_AT_JAMES)
+ .recipient(RECIPIENT1)
+ .mimeMessage(MimeMessageBuilder
+ .mimeMessageBuilder()
+ .addToRecipient(RECIPIENT2.toInternetAddress()))
+ .build();
+
+ assertThat(dlp.match(mail)).contains(RECIPIENT1);
+ }
+
+ @Test
+ void matchShouldReturnRecipientsWhenCcHeaderMatches() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder().recipientRule(Id.of("match recipient"), Pattern.compile(RECIPIENT2.asString())).build()));
+
+ FakeMail mail = FakeMail
+ .builder()
+ .sender(OTHER_AT_JAMES)
+ .recipient(RECIPIENT1)
+ .mimeMessage(MimeMessageBuilder
+ .mimeMessageBuilder()
+ .addCcRecipient(RECIPIENT2.toInternetAddress()))
+ .build();
+
+ assertThat(dlp.match(mail)).contains(RECIPIENT1);
+ }
+
+ @Test
+ void matchShouldReturnRecipientsWhenBccHeaderMatches() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder().recipientRule(Id.of("match recipient"), Pattern.compile(RECIPIENT2.asString())).build()));
+
+ FakeMail mail = FakeMail
+ .builder()
+ .sender(OTHER_AT_JAMES)
+ .recipient(RECIPIENT1)
+ .mimeMessage(MimeMessageBuilder
+ .mimeMessageBuilder()
+ .addBccRecipient(RECIPIENT2.toInternetAddress()))
+ .build();
+
+ assertThat(dlp.match(mail)).contains(RECIPIENT1);
+ }
+
+ @Test
+ void matchShouldReturnRecipientsWhenSubjectHeaderMatches() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder().contentRule(Id.of("match subject"), Pattern.compile("pony")).build()));
+
+ FakeMail mail = FakeMail
+ .builder()
+ .sender(OTHER_AT_JAMES)
+ .recipient(RECIPIENT1)
+ .mimeMessage(MimeMessageBuilder
+ .mimeMessageBuilder()
+ .setSubject("I just bought a pony"))
+ .build();
+
+ assertThat(dlp.match(mail)).contains(RECIPIENT1);
+ }
+
+ @Test
+ void matchShouldReturnRecipientsWhenMessageBodyMatches() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder().contentRule(Id.of("match content"), Pattern.compile("horse")).build()));
+
+ FakeMail mail = FakeMail
+ .builder()
+ .sender(OTHER_AT_JAMES)
+ .recipient(RECIPIENT1)
+ .mimeMessage(MimeMessageBuilder
+ .mimeMessageBuilder()
+ .setSubject("I just bought a pony")
+ .setText("It's actually a horse, not a pony"))
+ .build();
+
+ assertThat(dlp.match(mail)).contains(RECIPIENT1);
+ }
+
+ @Test
+ void matchShouldReturnRecipientsWhenMessageBodyMatchesWithNoSubject() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder().contentRule(Id.of("match content"), Pattern.compile("horse")).build()));
+
+ FakeMail mail = FakeMail
+ .builder()
+ .sender(OTHER_AT_JAMES)
+ .recipient(RECIPIENT1)
+ .mimeMessage(MimeMessageBuilder
+ .mimeMessageBuilder()
+ .setText("It's actually a horse, not a pony"))
+ .build();
+
+ assertThat(dlp.match(mail)).contains(RECIPIENT1);
+ }
+
+ @Test
+ void matchShouldReturnRecipientsWhenMessageMultipartBodyMatches() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder().contentRule(Id.of("match content"), Pattern.compile("horse")).build()));
+
+ FakeMail mail = FakeMail
+ .builder()
+ .sender(OTHER_AT_JAMES)
+ .recipient(RECIPIENT1)
+ .mimeMessage(MimeMessageBuilder
+ .mimeMessageBuilder()
+ .setSubject("I just bought a pony")
+ .setMultipartWithBodyParts(
+ MimeMessageBuilder.bodyPartBuilder()
+ .data("It's actually a donkey, not a pony"),
+ MimeMessageBuilder.bodyPartBuilder()
+ .data("What??? No it's a horse!!!")))
+ .build();
+
+ assertThat(dlp.match(mail)).contains(RECIPIENT1);
+ }
+
+ @Test
+ void matchShouldReturnRecipientsWhenEmbeddedMessageContentMatches() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder().contentRule(Id.of("match content"), Pattern.compile("horse")).build()));
+
+ FakeMail mail = FakeMail
+ .builder()
+ .sender(OTHER_AT_JAMES)
+ .recipient(RECIPIENT1)
+ .mimeMessage(MimeMessageBuilder
+ .mimeMessageBuilder()
+ .setSubject("I just bought a pony")
+ .setContent(
+ MimeMessageBuilder.multipartBuilder()
+ .addBody(
+ MimeMessageBuilder.bodyPartBuilder()
+ .data("It's actually a donkey, not a pony"))
+ .addBody(
+ MimeMessageBuilder.mimeMessageBuilder()
+ .setSender(RECIPIENT2.asString())
+ .setSubject("Embedded message with truth")
+ .setText("What??? No it's a horse!!!"))))
+ .build();
+
+ assertThat(dlp.match(mail)).contains(RECIPIENT1);
+ }
+
+ @Test
+ void matchShouldReturnEmptyWhenEmbeddedSenderMatchesInSubMessage() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder()
+ .senderRule(Id.of("match content"), Pattern.compile(RECIPIENT2.asString()))
+ .build()));
+
+ FakeMail mail = FakeMail
+ .builder()
+ .sender(OTHER_AT_JAMES)
+ .recipient(RECIPIENT1)
+ .mimeMessage(MimeMessageBuilder
+ .mimeMessageBuilder()
+ .setSubject("I just bought a pony")
+ .setSender(RECIPIENT1.asString())
+ .setContent(
+ MimeMessageBuilder.multipartBuilder()
+ .addBody(
+ MimeMessageBuilder.bodyPartBuilder()
+ .data("It's actually a donkey, not a pony"))
+ .addBody(
+ MimeMessageBuilder.mimeMessageBuilder()
+ .setSender(RECIPIENT2.asString())
+ .setSubject("Embedded message with truth")
+ .setText("What??? No it's a horse!!!"))))
+ .build();
+
+ assertThat(dlp.match(mail)).isEmpty();
+ }
+
+ @Test
+ void matchShouldReturnEmptyWhenEmbeddedRecipientMatchesInSubMessage() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder()
+ .recipientRule(Id.of("match content"), Pattern.compile(RECIPIENT2.asString()))
+ .build()));
+
+ FakeMail mail = FakeMail
+ .builder()
+ .sender(OTHER_AT_JAMES)
+ .recipient(RECIPIENT1)
+ .mimeMessage(MimeMessageBuilder
+ .mimeMessageBuilder()
+ .setSubject("I just bought a pony")
+ .setSender(RECIPIENT1.asString())
+ .setContent(
+ MimeMessageBuilder.multipartBuilder()
+ .addBody(
+ MimeMessageBuilder.bodyPartBuilder()
+ .data("It's actually a donkey, not a pony"))
+ .addBody(
+ MimeMessageBuilder.mimeMessageBuilder()
+ .addToRecipient(RECIPIENT1.asString())
+ .setSubject("Embedded message with truth")
+ .setText("What??? No it's a horse!!!"))))
+ .build();
+
+ assertThat(dlp.match(mail)).isEmpty();
+ }
+
+ @Test
+ void matchShouldReturnRecipientsWhenEncodedSubjectMatchesContentRule() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder()
+ .contentRule(Id.of("match content"), Pattern.compile("poné"))
+ .build()));
+
+ FakeMail mail = FakeMail
+ .builder()
+ .sender(OTHER_AT_JAMES)
+ .recipient(RECIPIENT1)
+ .mimeMessage(MimeMessageBuilder
+ .mimeMessageBuilder()
+ .setSubject("=?ISO-8859-1?Q?I_just_bought_a_pon=E9?=")
+ .setSender(RECIPIENT1.asString())
+ .setText("Meaningless text"))
+ .build();
+
+ assertThat(dlp.match(mail)).containsOnly(RECIPIENT1);
+ }
+
+ @Test
+ void matchShouldReturnRecipientsWhenEncodedTextMatchesContentRule() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder()
+ .contentRule(Id.of("match content"), Pattern.compile("poné"))
+ .build()));
+
+ FakeMail mail = FakeMail
+ .builder()
+ .sender(OTHER_AT_JAMES)
+ .recipient(RECIPIENT1)
+ .mimeMessage(MimeMessageBuilder
+ .mimeMessageBuilder()
+ .setSubject("you know what ?")
+ .setSender(RECIPIENT1.asString())
+ .setText("I bought a poné", "text/plain; charset=" + StandardCharsets.ISO_8859_1.name()))
+ .build();
+
+ assertThat(dlp.match(mail)).containsOnly(RECIPIENT1);
+ }
+
+ @Test
+ void matchShouldReturnRecipientsWhenRulesMatchesAMailboxRecipient() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder()
+ .recipientRule(Id.of("id1"), Pattern.compile(RECIPIENT1.asString()))
+ .build()));
+
+ MimeMessageBuilder meaninglessText = MimeMessageBuilder
+ .mimeMessageBuilder()
+ .addToRecipient("Name <" + RECIPIENT1.asString() + " >")
+ .setSubject("=?ISO-8859-1?Q?I_just_bought_a_pon=E9?=")
+ .setText("Meaningless text");
+
+ FakeMail mail = FakeMail
+ .builder()
+ .sender(OTHER_AT_JAMES)
+ .recipient(RECIPIENT2)
+ .mimeMessage(meaninglessText)
+ .build();
+
+ assertThat(dlp.match(mail)).containsOnly(RECIPIENT2);
+ }
+
+ @Test
+ void matchShouldReturnRecipientsWhenRulesMatchesAQuotedPrintableRecipient() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder()
+ .recipientRule(Id.of("id1"), Pattern.compile("Benoît"))
+ .build()));
+
+ MimeMessageBuilder meaninglessText = MimeMessageBuilder
+ .mimeMessageBuilder()
+ .addToRecipient("=?ISO-8859-1?Q?Beno=EEt_TELLIER?=")
+ .setSubject("=?ISO-8859-1?Q?I_just_bought_a_pon=E9?=")
+ .setText("Meaningless text");
+
+ FakeMail mail = FakeMail
+ .builder()
+ .sender(OTHER_AT_JAMES)
+ .recipient(RECIPIENT2)
+ .mimeMessage(meaninglessText)
+ .build();
+
+ assertThat(dlp.match(mail)).containsOnly(RECIPIENT2);
+ }
+
+ @Test
+ void matchShouldReturnRecipientsWhenRulesMatchesAQuotedPrintableSender() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder()
+ .senderRule(Id.of("id1"), Pattern.compile("Benoît"))
+ .build()));
+
+ MimeMessageBuilder meaninglessText = MimeMessageBuilder
+ .mimeMessageBuilder()
+ .addFrom("=?ISO-8859-1?Q?Beno=EEt_TELLIER?=")
+ .setSubject("=?ISO-8859-1?Q?I_just_bought_a_pon=E9?=")
+ .setText("Meaningless text");
+
+ FakeMail mail = FakeMail
+ .builder()
+ .sender(OTHER_AT_JAMES)
+ .recipient(RECIPIENT2)
+ .mimeMessage(meaninglessText)
+ .build();
+
+ assertThat(dlp.match(mail)).containsOnly(RECIPIENT2);
+ }
+
+ @Test
+ void matchShouldAttachMatchingRuleNameToMail() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder()
+ .recipientRule(Id.of("should not match recipient"), Pattern.compile(RECIPIENT3.asString()))
+ .senderRule(Id.of("should match sender"), Pattern.compile(JAMES_APACHE_ORG))
+ .build()));
+
+ FakeMail mail = FakeMail.builder()
+ .sender(ANY_AT_JAMES)
+ .recipient(RECIPIENT1)
+ .recipient(RECIPIENT2)
+ .build();
+
+ dlp.match(mail);
+
+ assertThat(mail.getAttribute("DlpMatchedRule")).isEqualTo("should match sender");
+ }
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org