You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2018/03/27 08:19:08 UTC
[24/41] james-project git commit: JAMES-2362 MessageID should be
indexed in ElasticSearch
JAMES-2362 MessageID should be indexed in ElasticSearch
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/93f1d3ea
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/93f1d3ea
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/93f1d3ea
Branch: refs/heads/master
Commit: 93f1d3eac6717fa27c665cea5294a37042d09a74
Parents: d8113c1
Author: benwa <bt...@linagora.com>
Authored: Tue Mar 27 09:39:10 2018 +0700
Committer: benwa <bt...@linagora.com>
Committed: Tue Mar 27 15:16:32 2018 +0700
----------------------------------------------------------------------
.../elasticsearch/MailboxMappingFactory.java | 6 ++++
.../elasticsearch/json/HeaderCollection.java | 17 ++++++++--
.../elasticsearch/json/IndexableMessage.java | 14 +++++++--
.../json/JsonMessageConstants.java | 1 +
.../json/HeaderCollectionTest.java | 33 ++++++++++++++++++++
...NonIndexableAttachmentWithoutAttachment.json | 1 +
.../store/src/test/resources/eml/htmlMail.json | 1 +
mailbox/store/src/test/resources/eml/mail.json | 1 +
.../src/test/resources/eml/nonTextual.json | 1 +
.../src/test/resources/eml/pgpSignedMail.json | 1 +
.../src/test/resources/eml/recursiveMail.json | 1 +
.../eml/recursiveMailWithoutAttachments.json | 1 +
.../store/src/test/resources/eml/spamMail.json | 1 +
13 files changed, 74 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxMappingFactory.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxMappingFactory.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxMappingFactory.java
index 506b73c..ac75a5b 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxMappingFactory.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxMappingFactory.java
@@ -53,6 +53,7 @@ import static org.apache.james.mailbox.elasticsearch.json.JsonMessageConstants.I
import static org.apache.james.mailbox.elasticsearch.json.JsonMessageConstants.MAILBOX_ID;
import static org.apache.james.mailbox.elasticsearch.json.JsonMessageConstants.MEDIA_TYPE;
import static org.apache.james.mailbox.elasticsearch.json.JsonMessageConstants.MESSAGE_ID;
+import static org.apache.james.mailbox.elasticsearch.json.JsonMessageConstants.MIME_MESSAGE_ID;
import static org.apache.james.mailbox.elasticsearch.json.JsonMessageConstants.MODSEQ;
import static org.apache.james.mailbox.elasticsearch.json.JsonMessageConstants.SENT_DATE;
import static org.apache.james.mailbox.elasticsearch.json.JsonMessageConstants.SIZE;
@@ -278,6 +279,11 @@ public class MailboxMappingFactory {
.field(INDEX, NOT_ANALYZED)
.endObject()
+ .startObject(MIME_MESSAGE_ID)
+ .field(TYPE, STRING)
+ .field(INDEX, NOT_ANALYZED)
+ .endObject()
+
.startObject(USERS)
.field(TYPE, STRING)
.field(INDEX, NOT_ANALYZED)
http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollection.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollection.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollection.java
index d50b1e2..4782f74 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollection.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollection.java
@@ -54,6 +54,7 @@ public class HeaderCollection {
private final Set<String> subjectSet;
private final Multimap<String, String> headers;
private Optional<ZonedDateTime> sentDate;
+ private Optional<String> messageID;
private Builder() {
toAddressSet = new HashSet<>();
@@ -64,6 +65,7 @@ public class HeaderCollection {
subjectSet = new HashSet<>();
headers = ArrayListMultimap.create();
sentDate = Optional.empty();
+ messageID = Optional.empty();
}
public Builder add(Field field) {
@@ -87,7 +89,7 @@ public class HeaderCollection {
ImmutableSet.copyOf(replyToAddressSet),
ImmutableSet.copyOf(subjectSet),
ImmutableMultimap.copyOf(headers),
- sentDate);
+ sentDate, messageID);
}
private void handleSpecificHeader(String headerName, String headerValue) {
@@ -105,6 +107,9 @@ public class HeaderCollection {
case DATE:
sentDate = SentDateComparator.toISODate(headerValue);
break;
+ case MESSAGE_ID:
+ messageID = Optional.ofNullable(headerValue);
+ break;
}
}
@@ -150,6 +155,7 @@ public class HeaderCollection {
public static final String REPLY_TO = "reply-to";
public static final String SUBJECT = "subject";
public static final String DATE = "date";
+ public static final String MESSAGE_ID = "message-id";
public static Builder builder() {
return new Builder();
@@ -163,10 +169,11 @@ public class HeaderCollection {
private final ImmutableSet<String> subjectSet;
private final ImmutableMultimap<String, String> headers;
private final Optional<ZonedDateTime> sentDate;
+ private final Optional<String> messageID;
private HeaderCollection(ImmutableSet<EMailer> toAddressSet, ImmutableSet<EMailer> fromAddressSet,
- ImmutableSet<EMailer> ccAddressSet, ImmutableSet<EMailer> bccAddressSet, ImmutableSet<EMailer> replyToAddressSet, ImmutableSet<String> subjectSet,
- ImmutableMultimap<String, String> headers, Optional<ZonedDateTime> sentDate) {
+ ImmutableSet<EMailer> ccAddressSet, ImmutableSet<EMailer> bccAddressSet, ImmutableSet<EMailer> replyToAddressSet, ImmutableSet<String> subjectSet,
+ ImmutableMultimap<String, String> headers, Optional<ZonedDateTime> sentDate, Optional<String> messageID) {
this.toAddressSet = toAddressSet;
this.fromAddressSet = fromAddressSet;
this.ccAddressSet = ccAddressSet;
@@ -175,6 +182,7 @@ public class HeaderCollection {
this.subjectSet = subjectSet;
this.headers = headers;
this.sentDate = sentDate;
+ this.messageID = messageID;
}
public Set<EMailer> getToAddressSet() {
@@ -209,4 +217,7 @@ public class HeaderCollection {
return headers;
}
+ public Optional<String> getMessageID() {
+ return messageID;
+ }
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java
index bd40558..ffcb140 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java
@@ -140,6 +140,7 @@ public class IndexableMessage {
EMailers cc = EMailers.from(headerCollection.getCcAddressSet());
EMailers bcc = EMailers.from(headerCollection.getBccAddressSet());
String sentDate = DateResolutionFormater.DATE_TIME_FOMATTER.format(headerCollection.getSentDate().orElse(internalDate));
+ Optional<String> mimeMessageID = headerCollection.getMessageID();
String text = Stream.of(from.serialize(),
to.serialize(),
@@ -197,7 +198,8 @@ public class IndexableMessage {
to,
uid,
userFlags,
- stringifiedUsers);
+ stringifiedUsers,
+ mimeMessageID);
}
private List<MimePart> setFlattenedAttachments(MimePart parsingResult, IndexAttachments indexAttachments) {
@@ -246,6 +248,7 @@ public class IndexableMessage {
private final long uid;
private final String[] userFlags;
private final List<String> users;
+ private final Optional<String> mimeMessageID;
private IndexableMessage(
List<MimePart> attachments,
@@ -277,7 +280,8 @@ public class IndexableMessage {
EMailers to,
long uid,
String[] userFlags,
- List<String> users) {
+ List<String> users,
+ Optional<String> mimeMessageID) {
this.attachments = attachments;
this.bcc = bcc;
this.bodyHtml = bodyHtml;
@@ -308,6 +312,7 @@ public class IndexableMessage {
this.uid = uid;
this.userFlags = userFlags;
this.users = users;
+ this.mimeMessageID = mimeMessageID;
}
@JsonProperty(JsonMessageConstants.ATTACHMENTS)
@@ -459,4 +464,9 @@ public class IndexableMessage {
public boolean isUnRead() {
return isUnRead;
}
+
+ @JsonProperty(JsonMessageConstants.MIME_MESSAGE_ID)
+ public Optional<String> getMimeMessageID() {
+ return mimeMessageID;
+ }
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java
index f931051..d6fa74e 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java
@@ -46,6 +46,7 @@ public interface JsonMessageConstants {
String SENT_DATE = "sentDate";
String ATTACHMENTS = "attachments";
String TEXT = "text";
+ String MIME_MESSAGE_ID = "mimeMessageID";
/*
James properties we can easily get
http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollectionTest.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollectionTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollectionTest.java
index 52b4002..9d10e7a 100644
--- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollectionTest.java
+++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollectionTest.java
@@ -272,6 +272,39 @@ public class HeaderCollectionTest {
assertThat(headerCollection.getSubjectSet()).containsOnly("A fantastic ElasticSearch module will be available soon for JAMES");
}
+ @Test
+ public void getMessageIDShouldReturnMessageIdValue() {
+ String messageID = "<ab...@123>";
+ HeaderCollection headerCollection = HeaderCollection.builder()
+ .add(new FieldImpl("Message-ID", messageID))
+ .build();
+
+ assertThat(headerCollection.getMessageID())
+ .contains(messageID);
+ }
+
+ @Test
+ public void getMessageIDShouldReturnLatestEncounteredMessageIdValue() {
+ String messageID = "<ab...@123>";
+ HeaderCollection headerCollection = HeaderCollection.builder()
+ .add(new FieldImpl("Message-ID", "<ot...@toto.com>"))
+ .add(new FieldImpl("Message-ID", messageID))
+ .build();
+
+ assertThat(headerCollection.getMessageID())
+ .contains(messageID);
+ }
+
+ @Test
+ public void getMessageIDShouldReturnEmptyWhenNoMessageId() {
+ HeaderCollection headerCollection = HeaderCollection.builder()
+ .add(new FieldImpl("Other", "value"))
+ .build();
+
+ assertThat(headerCollection.getMessageID())
+ .isEmpty();
+ }
+
@Test(expected = NullPointerException.class)
public void nullFieldShouldThrow() {
HeaderCollection.builder().add(null).build();
http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/store/src/test/resources/eml/emailWithNonIndexableAttachmentWithoutAttachment.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/emailWithNonIndexableAttachmentWithoutAttachment.json b/mailbox/store/src/test/resources/eml/emailWithNonIndexableAttachmentWithoutAttachment.json
index deadebc..b960a84 100644
--- a/mailbox/store/src/test/resources/eml/emailWithNonIndexableAttachmentWithoutAttachment.json
+++ b/mailbox/store/src/test/resources/eml/emailWithNonIndexableAttachmentWithoutAttachment.json
@@ -13,6 +13,7 @@
"mailboxId" : "18",
"date" : "2016-12-07T11:51:51+0100",
"size" : 25,
+ "mimeMessageID": "1480502737913",
"properties" : [
{
"value" : "plain",
http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/store/src/test/resources/eml/htmlMail.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/htmlMail.json b/mailbox/store/src/test/resources/eml/htmlMail.json
index a8998ef..04cc4e3 100644
--- a/mailbox/store/src/test/resources/eml/htmlMail.json
+++ b/mailbox/store/src/test/resources/eml/htmlMail.json
@@ -8,6 +8,7 @@
"mediaType":"plain",
"subtype":"text",
"userFlags":["social","pocket-money"],
+ "mimeMessageID": "<55...@i-dee0850e.mail>",
"headers":{
"date":[
"Thu, 04 Jun 2015 07:36:08 +0000"
http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/store/src/test/resources/eml/mail.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/mail.json b/mailbox/store/src/test/resources/eml/mail.json
index 76068a5..2ad96b7 100644
--- a/mailbox/store/src/test/resources/eml/mail.json
+++ b/mailbox/store/src/test/resources/eml/mail.json
@@ -11,6 +11,7 @@
"security",
"debian"
],
+ "mimeMessageID": "<CA...@mail.gmail.com>",
"headers": {
"mime-version": [
"1.0"
http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/store/src/test/resources/eml/nonTextual.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/nonTextual.json b/mailbox/store/src/test/resources/eml/nonTextual.json
index af831cb..c9f3ed9 100644
--- a/mailbox/store/src/test/resources/eml/nonTextual.json
+++ b/mailbox/store/src/test/resources/eml/nonTextual.json
@@ -8,6 +8,7 @@
"mediaType":"plain",
"subtype":"text",
"userFlags":[],
+ "mimeMessageID": "<55...@linagora.com>",
"headers":{
"date":[
"Thu, 18 Jun 2015 12:43:26 +0200"
http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/store/src/test/resources/eml/pgpSignedMail.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/pgpSignedMail.json b/mailbox/store/src/test/resources/eml/pgpSignedMail.json
index 0b00109..b565d75 100644
--- a/mailbox/store/src/test/resources/eml/pgpSignedMail.json
+++ b/mailbox/store/src/test/resources/eml/pgpSignedMail.json
@@ -11,6 +11,7 @@
"security",
"debian"
],
+ "mimeMessageID": "<E1...@master.debian.org>",
"headers": {
"date": [
"Wed, 03 Jun 2015 19:14:32 +0000"
http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/store/src/test/resources/eml/recursiveMail.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/recursiveMail.json b/mailbox/store/src/test/resources/eml/recursiveMail.json
index f0ae82e..ba1ca5c 100644
--- a/mailbox/store/src/test/resources/eml/recursiveMail.json
+++ b/mailbox/store/src/test/resources/eml/recursiveMail.json
@@ -11,6 +11,7 @@
"security",
"debian"
],
+ "mimeMessageID": "<55...@linagora.com>",
"headers": {
"date": [
"Wed, 10 Jun 2015 10:45:27 +0200"
http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json b/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json
index 9dfb964..c40110f 100644
--- a/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json
+++ b/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json
@@ -7,6 +7,7 @@
"date": "2015-06-07T00:00:00+02:00",
"mediaType": "plain",
"subtype": "text",
+ "mimeMessageID": "<55...@linagora.com>",
"userFlags": [
"security",
"debian"
http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/store/src/test/resources/eml/spamMail.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/spamMail.json b/mailbox/store/src/test/resources/eml/spamMail.json
index 388aa2d..d133cd7 100644
--- a/mailbox/store/src/test/resources/eml/spamMail.json
+++ b/mailbox/store/src/test/resources/eml/spamMail.json
@@ -7,6 +7,7 @@
"date": "2015-06-07T00:00:00+0200",
"mediaType": "plain",
"subtype": "text",
+ "mimeMessageID": "<VA...@spam.minet.net>",
"userFlags": [],
"headers": {
"mime-version": [
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org