You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2020/05/04 03:21:22 UTC
[james-project] 03/12: JAMES-2997 TextExtractor should rely on
ContentType
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit e7de644d3e5ff95f69ccdd47a768e738e01a1a31
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Mon Apr 27 19:34:27 2020 +0700
JAMES-2997 TextExtractor should rely on ContentType
These classes were not handling the full content field but expecting the
MimeType alone.
Furthermore, we can take advantage og having the charset available, ie
when calling tika.
---
.../james/mailbox/extractor/TextExtractor.java | 4 +-
.../apache/james/mailbox/model/ContentType.java | 179 ++++++++++++++++++++-
.../james/mailbox/model/ContentTypeTest.java | 179 +++++++++++++++++++++
.../james/mailbox/elasticsearch/json/MimePart.java | 35 ++--
.../json/MimePartContainerBuilder.java | 6 +-
.../mailbox/elasticsearch/json/MimePartParser.java | 12 +-
.../json/RootMimePartContainerBuilder.java | 6 +-
...asticSearchListeningMessageSearchIndexTest.java | 3 +-
.../mailbox/elasticsearch/json/MimePartTest.java | 11 +-
.../mailbox/store/search/PDFTextExtractor.java | 10 +-
.../mailbox/store/search/PDFTextExtractorTest.java | 7 +-
.../store/extractor/DefaultTextExtractor.java | 5 +-
.../store/extractor/JsoupTextExtractor.java | 10 +-
.../mailbox/store/search/MessageSearches.java | 3 +-
.../store/extractor/DefaultTextExtractorTest.java | 14 +-
.../store/extractor/JsoupTextExtractorTest.java | 33 +++-
.../test/resources/documents/html-iso-8859-1.html} | 25 +--
.../resources/documents/simple-text-iso-8859-1.txt | 1 +
.../james/mailbox/tika/CachingTextExtractor.java | 5 +-
.../tika/ContentTypeFilteringTextExtractor.java | 12 +-
.../james/mailbox/tika/TikaConfiguration.java | 11 +-
.../apache/james/mailbox/tika/TikaHttpClient.java | 4 +-
.../james/mailbox/tika/TikaHttpClientImpl.java | 9 +-
.../james/mailbox/tika/TikaTextExtractor.java | 8 +-
.../mailbox/tika/CachingTextExtractorTest.java | 3 +-
.../ContentTypeFilteringTextExtractorTest.java | 22 ++-
.../james/mailbox/tika/TikaTextExtractorTest.java | 38 +++--
.../modules/mailbox/TikaConfigurationReader.java | 4 +-
.../mailbox/TikaConfigurationReaderTest.java | 5 +-
.../jmap/draft/methods/MIMEMessageConverter.java | 4 +-
30 files changed, 567 insertions(+), 101 deletions(-)
diff --git a/mailbox/api/src/main/java/org/apache/james/mailbox/extractor/TextExtractor.java b/mailbox/api/src/main/java/org/apache/james/mailbox/extractor/TextExtractor.java
index b33d853..6b91122 100644
--- a/mailbox/api/src/main/java/org/apache/james/mailbox/extractor/TextExtractor.java
+++ b/mailbox/api/src/main/java/org/apache/james/mailbox/extractor/TextExtractor.java
@@ -21,8 +21,10 @@ package org.apache.james.mailbox.extractor;
import java.io.InputStream;
+import org.apache.james.mailbox.model.ContentType;
+
public interface TextExtractor {
- ParsedContent extractContent(InputStream inputStream, String contentType) throws Exception;
+ ParsedContent extractContent(InputStream inputStream, ContentType contentType) throws Exception;
}
diff --git a/mailbox/api/src/main/java/org/apache/james/mailbox/model/ContentType.java b/mailbox/api/src/main/java/org/apache/james/mailbox/model/ContentType.java
index ed8ecf5..5eeeb26 100644
--- a/mailbox/api/src/main/java/org/apache/james/mailbox/model/ContentType.java
+++ b/mailbox/api/src/main/java/org/apache/james/mailbox/model/ContentType.java
@@ -19,15 +19,168 @@
package org.apache.james.mailbox.model;
+import java.io.StringReader;
+import java.nio.charset.Charset;
+import java.util.Objects;
+import java.util.Optional;
+
+import org.apache.james.mime4j.dom.field.ContentTypeField;
+import org.apache.james.mime4j.field.Fields;
+import org.apache.james.mime4j.field.contenttype.parser.ContentTypeParser;
+import org.apache.james.mime4j.field.contenttype.parser.ParseException;
+
+import com.google.common.base.MoreObjects;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
public class ContentType {
+ public static class MimeType {
+ public static MimeType of(String value) {
+ ContentTypeParser parser = new ContentTypeParser(new StringReader(value));
+ try {
+ parser.parseAll();
+ } catch (ParseException e) {
+ throw new IllegalArgumentException("Invalid mimeType", e);
+ }
+ return new MimeType(
+ new MediaType(parser.getType()),
+ new SubType(parser.getSubType()));
+ }
+
+ public static MimeType of(MediaType mediaType, SubType subType) {
+ return new MimeType(mediaType, subType);
+ }
+
+ private final MediaType mediaType;
+ private final SubType subType;
+
+ private MimeType(MediaType mediaType, SubType subType) {
+ this.mediaType = mediaType;
+ this.subType = subType;
+ }
+
+ public String asString() {
+ return mediaType.asString() + "/" + subType.asString();
+ }
+
+ @Override
+ public final boolean equals(Object o) {
+ if (o instanceof MimeType) {
+ MimeType mimeType = (MimeType) o;
+
+ return Objects.equals(this.mediaType, mimeType.mediaType)
+ && Objects.equals(this.subType, mimeType.subType);
+ }
+ return false;
+ }
+
+ @Override
+ public final int hashCode() {
+ return Objects.hash(mediaType, subType);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("mediaType", mediaType)
+ .add("subType", subType)
+ .toString();
+ }
+ }
+
+ public static class MediaType {
+ public static MediaType of(String value) {
+ Preconditions.checkState(!Strings.isNullOrEmpty(value), "'media type' is mandatory");
+ return new MediaType(value);
+ }
+
+ private final String value;
+
+ private MediaType(String value) {
+ this.value = value;
+ }
+
+ public String asString() {
+ return value;
+ }
+
+ @Override
+ public final boolean equals(Object o) {
+ if (o instanceof MediaType) {
+ MediaType mediaType = (MediaType) o;
+
+ return Objects.equals(this.value, mediaType.value);
+ }
+ return false;
+ }
+
+ @Override
+ public final int hashCode() {
+ return Objects.hash(value);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("value", value)
+ .toString();
+ }
+ }
+
+ public static class SubType {
+ public static SubType of(String value) {
+ Preconditions.checkState(!Strings.isNullOrEmpty(value), "'sub type' is mandatory");
+ return new SubType(value);
+ }
+
+ private final String value;
+
+ private SubType(String value) {
+ this.value = value;
+ }
+
+ public String asString() {
+ return value;
+ }
+
+ @Override
+ public final boolean equals(Object o) {
+ if (o instanceof SubType) {
+ SubType subType = (SubType) o;
+
+ return Objects.equals(this.value, subType.value);
+ }
+ return false;
+ }
+
+ @Override
+ public final int hashCode() {
+ return Objects.hash(value);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("value", value)
+ .toString();
+ }
+ }
+
public static ContentType of(String value) {
- Preconditions.checkState(!Strings.isNullOrEmpty(value), "'content type' is mandatory");
+ Preconditions.checkArgument(!Strings.isNullOrEmpty(value), "'content type' is mandatory");
return new ContentType(value);
}
+ public static ContentType of(MimeType mimeType) {
+ return new ContentType(mimeType.asString());
+ }
+
+ public static ContentType of(MimeType mimeType, Optional<Charset> charset) {
+ return ContentType.of(
+ charset.map(value -> mimeType.asString() + "; charset=" + value.name())
+ .orElse(mimeType.asString()));
+ }
+
/**
* Follows syntax and usage as defined in https://tools.ietf.org/html/rfc2045#section-5
* Thus includes media type and parameters, including charset
@@ -39,6 +192,30 @@ public class ContentType {
this.value = value;
}
+ public ContentTypeField asMime4J() {
+ return Fields.contentType(value);
+ }
+
+ public MimeType mimeType() {
+ ContentTypeField contentTypeField = asMime4J();
+ return MimeType.of(
+ MediaType.of(contentTypeField.getMediaType()),
+ SubType.of(contentTypeField.getSubType()));
+ }
+
+ public MediaType mediaType() {
+ return MediaType.of(asMime4J().getMediaType());
+ }
+
+ public SubType subType() {
+ return SubType.of(asMime4J().getSubType());
+ }
+
+ public Optional<Charset> charset() {
+ return Optional.ofNullable(asMime4J().getCharset())
+ .map(Charset::forName);
+ }
+
public String asString() {
return value;
}
diff --git a/mailbox/api/src/test/java/org/apache/james/mailbox/model/ContentTypeTest.java b/mailbox/api/src/test/java/org/apache/james/mailbox/model/ContentTypeTest.java
new file mode 100644
index 0000000..d8568bb
--- /dev/null
+++ b/mailbox/api/src/test/java/org/apache/james/mailbox/model/ContentTypeTest.java
@@ -0,0 +1,179 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.mailbox.model;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatCode;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.nio.charset.StandardCharsets;
+
+import org.junit.jupiter.api.Test;
+
+import nl.jqno.equalsverifier.EqualsVerifier;
+
+class ContentTypeTest {
+ @Test
+ void contentTypeShouldRespectBeanContract() {
+ EqualsVerifier.forClass(ContentType.class)
+ .verify();
+ }
+
+ @Test
+ void subTypeShouldRespectBeanContract() {
+ EqualsVerifier.forClass(ContentType.SubType.class)
+ .verify();
+ }
+
+ @Test
+ void mediaTypeShouldRespectBeanContract() {
+ EqualsVerifier.forClass(ContentType.MediaType.class)
+ .verify();
+ }
+
+ @Test
+ void mimeTypeShouldRespectBeanContract() {
+ EqualsVerifier.forClass(ContentType.MimeType.class)
+ .verify();
+ }
+
+ @Test
+ void mimeTypeOfShouldThrowWhenInvalid() {
+ assertThatThrownBy(() -> ContentType.MimeType.of("aaa"))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void mimeTypeOfShouldThrowWhenEmpty() {
+ assertThatThrownBy(() -> ContentType.of(""))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void mimeTypeOfShouldThrowWhenNull() {
+ assertThatThrownBy(() -> ContentType.of((String) null))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void mimeTypeOfShouldReturnExpectedValue() {
+ assertThat(ContentType.MimeType.of("text/html"))
+ .isEqualTo(ContentType.MimeType.of(
+ ContentType.MediaType.of("text"),
+ ContentType.SubType.of("html")));
+ }
+
+ @Test
+ void mediaTypeOfShouldThrowWhenEmpty() {
+ assertThatThrownBy(() -> ContentType.MediaType.of(""))
+ .isInstanceOf(IllegalStateException.class);
+ }
+
+ @Test
+ void subTypeOfShouldThrowWhenEmpty() {
+ assertThatThrownBy(() -> ContentType.SubType.of(""))
+ .isInstanceOf(IllegalStateException.class);
+ }
+
+ @Test
+ void asMime4JShouldNotThrowWhenInvalidContentType() {
+ ContentType invalid = ContentType.of("/invalid");
+ assertThatCode(invalid::asMime4J)
+ .doesNotThrowAnyException();
+ }
+
+ @Test
+ void mimeTypeShouldThrowWhenInvalidContentType() {
+ ContentType invalid = ContentType.of("invalid");
+ assertThatThrownBy(invalid::mimeType)
+ .isInstanceOf(IllegalStateException.class);
+ }
+
+ @Test
+ void mediaTypeShouldThrowWhenInvalidContentType() {
+ ContentType invalid = ContentType.of("invalid");
+ assertThatThrownBy(invalid::mediaType)
+ .isInstanceOf(IllegalStateException.class);
+ }
+
+ @Test
+ void subTypeShouldThrowWhenInvalidContentType() {
+ ContentType invalid = ContentType.of("invalid");
+ assertThatThrownBy(invalid::subType)
+ .isInstanceOf(IllegalStateException.class);
+ }
+
+ @Test
+ void charsetShouldThrowWhenInvalidCharset() {
+ ContentType invalid = ContentType.of("text/plain; charset=invalid");
+ assertThatThrownBy(invalid::charset)
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void charsetShouldThrowWhenEmptyCharset() {
+ ContentType contentType = ContentType.of("text/plain; charset=");
+
+ assertThat(contentType.charset())
+ .isEmpty();
+ }
+
+ @Test
+ void asStringShouldReturnWhenInvalid() {
+ String value = "invalid";
+ ContentType invalid = ContentType.of(value);
+ assertThat(invalid.asString()).isEqualTo(value);
+ }
+
+ @Test
+ void charsetShouldReturnEmptyWhenNone() {
+ ContentType contentType = ContentType.of("text/html");
+
+ assertThat(contentType.charset()).isEmpty();
+ }
+
+ @Test
+ void charsetShouldReturnSuppliedCharset() {
+ ContentType contentType = ContentType.of("text/html; charset=UTF-8");
+
+ assertThat(contentType.charset()).contains(StandardCharsets.UTF_8);
+ }
+
+ @Test
+ void mimeTypeShouldReturnSuppliedValue() {
+ ContentType contentType = ContentType.of("text/html");
+
+ assertThat(contentType.mimeType()).isEqualTo(ContentType.MimeType.of("text/html"));
+ }
+
+ @Test
+ void subTypeShouldReturnSuppliedValue() {
+ ContentType contentType = ContentType.of("text/html");
+
+ assertThat(contentType.subType()).isEqualTo(ContentType.SubType.of("html"));
+ }
+
+ @Test
+ void mediaTypeShouldReturnSuppliedValue() {
+ ContentType contentType = ContentType.of("text/html");
+
+ assertThat(contentType.mediaType()).isEqualTo(ContentType.MediaType.of("text"));
+ }
+}
\ No newline at end of file
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePart.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePart.java
index 55fe7b6..d818cfd 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePart.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePart.java
@@ -30,6 +30,9 @@ import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.apache.james.mailbox.extractor.ParsedContent;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.model.ContentType;
+import org.apache.james.mailbox.model.ContentType.MediaType;
+import org.apache.james.mailbox.model.ContentType.SubType;
import org.apache.james.mailbox.store.extractor.DefaultTextExtractor;
import org.apache.james.mime4j.stream.Field;
import org.slf4j.Logger;
@@ -48,8 +51,8 @@ public class MimePart {
private final HeaderCollection.Builder headerCollectionBuilder;
private Optional<InputStream> bodyContent;
private final List<MimePart> children;
- private Optional<String> mediaType;
- private Optional<String> subType;
+ private Optional<MediaType> mediaType;
+ private Optional<SubType> subType;
private Optional<String> fileName;
private Optional<String> fileExtension;
private Optional<String> contentDisposition;
@@ -95,14 +98,14 @@ public class MimePart {
}
@Override
- public Builder addMediaType(String mediaType) {
+ public Builder addMediaType(MediaType mediaType) {
this.mediaType = Optional.ofNullable(mediaType);
return this;
}
@Override
- public Builder addSubType(String subType) {
- this.subType = Optional.ofNullable(subType);
+ public Builder addSubType(SubType subType) {
+ this.subType = Optional.of(subType);
return this;
}
@@ -166,16 +169,18 @@ public class MimePart {
}
private Boolean isTextBody() {
- return mediaType.map("text"::equals).orElse(false);
+ return mediaType.map(MediaType.of("text")::equals).orElse(false);
}
private Boolean isHtml() {
- return isTextBody() && subType.map("html"::equals).orElse(false);
+ return isTextBody() && subType.map(SubType.of("html")::equals).orElse(false);
}
- private Optional<String> computeContentType() {
+ private Optional<ContentType> computeContentType() {
if (mediaType.isPresent() && subType.isPresent()) {
- return Optional.of(mediaType.get() + "/" + subType.get());
+ return Optional.of(ContentType.of(
+ ContentType.MimeType.of(mediaType.get(), subType.get()),
+ charset));
} else {
return Optional.empty();
}
@@ -191,15 +196,15 @@ public class MimePart {
private final HeaderCollection headerCollection;
private final Optional<String> bodyTextContent;
- private final Optional<String> mediaType;
- private final Optional<String> subType;
+ private final Optional<MediaType> mediaType;
+ private final Optional<SubType> subType;
private final Optional<String> fileName;
private final Optional<String> fileExtension;
private final Optional<String> contentDisposition;
private final List<MimePart> attachments;
- private MimePart(HeaderCollection headerCollection, Optional<String> bodyTextContent, Optional<String> mediaType,
- Optional<String> subType, Optional<String> fileName, Optional<String> fileExtension,
+ private MimePart(HeaderCollection headerCollection, Optional<String> bodyTextContent, Optional<MediaType> mediaType,
+ Optional<SubType> subType, Optional<String> fileName, Optional<String> fileExtension,
Optional<String> contentDisposition, List<MimePart> attachments) {
this.headerCollection = headerCollection;
this.mediaType = mediaType;
@@ -233,12 +238,12 @@ public class MimePart {
@JsonProperty(JsonMessageConstants.Attachment.MEDIA_TYPE)
public Optional<String> getMediaType() {
- return mediaType;
+ return mediaType.map(MediaType::asString);
}
@JsonProperty(JsonMessageConstants.Attachment.SUBTYPE)
public Optional<String> getSubType() {
- return subType;
+ return subType.map(SubType::asString);
}
@JsonProperty(JsonMessageConstants.Attachment.CONTENT_DISPOSITION)
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePartContainerBuilder.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePartContainerBuilder.java
index 4a71d21..88439fb 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePartContainerBuilder.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePartContainerBuilder.java
@@ -23,6 +23,8 @@ import java.io.InputStream;
import java.nio.charset.Charset;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.model.ContentType.MediaType;
+import org.apache.james.mailbox.model.ContentType.SubType;
import org.apache.james.mime4j.stream.Field;
public interface MimePartContainerBuilder {
@@ -41,9 +43,9 @@ public interface MimePartContainerBuilder {
MimePartContainerBuilder charset(Charset charset);
- MimePartContainerBuilder addMediaType(String mediaType);
+ MimePartContainerBuilder addMediaType(MediaType mediaType);
- MimePartContainerBuilder addSubType(String subType);
+ MimePartContainerBuilder addSubType(SubType subType);
MimePartContainerBuilder addContentDisposition(String contentDisposition);
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePartParser.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePartParser.java
index a87ae44..d12748f 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePartParser.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePartParser.java
@@ -26,6 +26,8 @@ import java.util.LinkedList;
import java.util.Optional;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.model.ContentType.MediaType;
+import org.apache.james.mailbox.model.ContentType.SubType;
import org.apache.james.mailbox.store.mail.model.Message;
import org.apache.james.mime4j.MimeException;
import org.apache.james.mime4j.message.DefaultBodyDescriptorBuilder;
@@ -119,9 +121,13 @@ public class MimePartParser {
private void extractMimePartBodyDescription(MimeTokenStream stream) {
MaximalBodyDescriptor descriptor = (MaximalBodyDescriptor) stream.getBodyDescriptor();
- currentlyBuildMimePart.addMediaType(descriptor.getMediaType())
- .addSubType(descriptor.getSubType())
- .addContentDisposition(descriptor.getContentDispositionType())
+ Optional.ofNullable(descriptor.getMediaType())
+ .map(MediaType::of)
+ .ifPresent(currentlyBuildMimePart::addMediaType);
+ Optional.ofNullable(descriptor.getSubType())
+ .map(SubType::of)
+ .ifPresent(currentlyBuildMimePart::addSubType);
+ currentlyBuildMimePart.addContentDisposition(descriptor.getContentDispositionType())
.addFileName(descriptor.getContentDispositionFilename());
extractCharset(descriptor);
}
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/RootMimePartContainerBuilder.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/RootMimePartContainerBuilder.java
index 81e58be..71a88ce 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/RootMimePartContainerBuilder.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/RootMimePartContainerBuilder.java
@@ -23,6 +23,8 @@ import java.io.InputStream;
import java.nio.charset.Charset;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.model.ContentType.MediaType;
+import org.apache.james.mailbox.model.ContentType.SubType;
import org.apache.james.mime4j.stream.Field;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -71,13 +73,13 @@ public class RootMimePartContainerBuilder implements MimePartContainerBuilder {
}
@Override
- public MimePartContainerBuilder addMediaType(String mediaType) {
+ public MimePartContainerBuilder addMediaType(MediaType mediaType) {
LOGGER.warn("Trying to add media type to the Root MimePart container");
return this;
}
@Override
- public MimePartContainerBuilder addSubType(String subType) {
+ public MimePartContainerBuilder addSubType(SubType subType) {
LOGGER.warn("Trying to add sub type to the Root MimePart container");
return this;
}
diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndexTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndexTest.java
index dcbeb0f..bd4fc78 100644
--- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndexTest.java
+++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndexTest.java
@@ -58,6 +58,7 @@ import org.apache.james.mailbox.inmemory.InMemoryMessageId;
import org.apache.james.mailbox.manager.ManagerTestProvisionner;
import org.apache.james.mailbox.model.AttachmentId;
import org.apache.james.mailbox.model.AttachmentMetadata;
+import org.apache.james.mailbox.model.ContentType;
import org.apache.james.mailbox.model.Mailbox;
import org.apache.james.mailbox.model.MailboxPath;
import org.apache.james.mailbox.model.MessageAttachmentMetadata;
@@ -130,7 +131,7 @@ class ElasticSearchListeningMessageSearchIndexTest {
static class FailingTextExtractor implements TextExtractor {
@Override
- public ParsedContent extractContent(InputStream inputStream, String contentType) {
+ public ParsedContent extractContent(InputStream inputStream, ContentType contentType) {
throw new RuntimeException();
}
}
diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MimePartTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MimePartTest.java
index 79a03c8..19fde86 100644
--- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MimePartTest.java
+++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MimePartTest.java
@@ -23,6 +23,9 @@ import static org.assertj.core.api.Assertions.assertThat;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
+import org.apache.james.mailbox.model.ContentType;
+import org.apache.james.mailbox.model.ContentType.MediaType;
+import org.apache.james.mailbox.model.ContentType.SubType;
import org.junit.jupiter.api.Test;
class MimePartTest {
@@ -31,8 +34,8 @@ class MimePartTest {
void buildShouldWorkWhenTextualContentFromParserIsEmpty() {
MimePart.builder()
.addBodyContent(new ByteArrayInputStream(new byte[] {}))
- .addMediaType("text")
- .addSubType("plain")
+ .addMediaType(MediaType.of("text"))
+ .addSubType(SubType.of("plain"))
.build();
}
@@ -41,8 +44,8 @@ class MimePartTest {
String body = "text";
MimePart mimePart = MimePart.builder()
.addBodyContent(new ByteArrayInputStream(body.getBytes(StandardCharsets.UTF_8)))
- .addMediaType("text")
- .addSubType("plain")
+ .addMediaType(MediaType.of("text"))
+ .addSubType(SubType.of("plain"))
.build();
assertThat(mimePart.getTextualBody()).contains(body);
diff --git a/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/PDFTextExtractor.java b/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/PDFTextExtractor.java
index 8a6f583..8f82685 100644
--- a/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/PDFTextExtractor.java
+++ b/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/PDFTextExtractor.java
@@ -26,6 +26,8 @@ import java.util.Optional;
import org.apache.commons.io.IOUtils;
import org.apache.james.mailbox.extractor.ParsedContent;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.model.ContentType;
+import org.apache.james.mailbox.model.ContentType.MimeType;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.text.PDFTextStripper;
@@ -34,10 +36,10 @@ import com.google.common.collect.ImmutableMap;
public class PDFTextExtractor implements TextExtractor {
- static final String PDF_TYPE = "application/pdf";
+ static final MimeType PDF_TYPE = MimeType.of("application/pdf");
@Override
- public ParsedContent extractContent(InputStream inputStream, String contentType) throws Exception {
+ public ParsedContent extractContent(InputStream inputStream, ContentType contentType) throws Exception {
Preconditions.checkNotNull(inputStream);
Preconditions.checkNotNull(contentType);
@@ -47,8 +49,8 @@ public class PDFTextExtractor implements TextExtractor {
return new ParsedContent(Optional.ofNullable(IOUtils.toString(inputStream, StandardCharsets.UTF_8)), ImmutableMap.of());
}
- private boolean isPDF(String contentType) {
- return contentType.equals(PDF_TYPE);
+ private boolean isPDF(ContentType contentType) {
+ return contentType.mimeType().equals(PDF_TYPE);
}
private ParsedContent extractTextFromPDF(InputStream inputStream) throws IOException {
diff --git a/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/PDFTextExtractorTest.java b/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/PDFTextExtractorTest.java
index 65c28fd..de4bac9 100644
--- a/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/PDFTextExtractorTest.java
+++ b/mailbox/scanning-search/src/test/java/org/apache/james/mailbox/store/search/PDFTextExtractorTest.java
@@ -26,6 +26,7 @@ import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
+import org.apache.james.mailbox.model.ContentType;
import org.junit.Before;
import org.junit.Test;
@@ -41,7 +42,7 @@ public class PDFTextExtractorTest {
@Test
public void extractContentShouldThrowWhenNullInputStream() throws Exception {
assertThatThrownBy(() ->
- testee.extractContent(null, "any/any"))
+ testee.extractContent(null, ContentType.of("any/any")))
.isInstanceOf(NullPointerException.class);
}
@@ -57,7 +58,7 @@ public class PDFTextExtractorTest {
String content = "content";
InputStream inputStream = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
- assertThat(testee.extractContent(inputStream, "text/plain")
+ assertThat(testee.extractContent(inputStream, ContentType.of("text/plain"))
.getTextualContent())
.contains(content);
}
@@ -67,7 +68,7 @@ public class PDFTextExtractorTest {
String content = "Little PDF\n";
InputStream inputStream = ClassLoader.getSystemResourceAsStream("pdf.pdf");
- assertThat(testee.extractContent(inputStream, PDFTextExtractor.PDF_TYPE)
+ assertThat(testee.extractContent(inputStream, ContentType.of(PDFTextExtractor.PDF_TYPE))
.getTextualContent())
.contains(content);
}
diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/extractor/DefaultTextExtractor.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/extractor/DefaultTextExtractor.java
index 23e574e..ea4d76b 100644
--- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/extractor/DefaultTextExtractor.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/extractor/DefaultTextExtractor.java
@@ -27,6 +27,7 @@ import java.util.Optional;
import org.apache.commons.io.IOUtils;
import org.apache.james.mailbox.extractor.ParsedContent;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.model.ContentType;
/**
* A default text extractor that is directly based on the input file provided.
@@ -36,8 +37,8 @@ import org.apache.james.mailbox.extractor.TextExtractor;
public class DefaultTextExtractor implements TextExtractor {
@Override
- public ParsedContent extractContent(InputStream inputStream, String contentType) throws Exception {
- if (contentType != null && contentType.startsWith("text/")) {
+ public ParsedContent extractContent(InputStream inputStream, ContentType contentType) throws Exception {
+ if (contentType != null && contentType.asString().startsWith("text/")) {
return new ParsedContent(Optional.ofNullable(IOUtils.toString(inputStream, StandardCharsets.UTF_8)), new HashMap<>());
} else {
return new ParsedContent(Optional.empty(), new HashMap<>());
diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/extractor/JsoupTextExtractor.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/extractor/JsoupTextExtractor.java
index 06b5d80..5404455 100644
--- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/extractor/JsoupTextExtractor.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/extractor/JsoupTextExtractor.java
@@ -29,6 +29,8 @@ import java.util.Optional;
import org.apache.commons.io.IOUtils;
import org.apache.james.mailbox.extractor.ParsedContent;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.model.ContentType;
+import org.apache.james.mailbox.model.ContentType.MimeType;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
@@ -38,16 +40,18 @@ public class JsoupTextExtractor implements TextExtractor {
private static final String TITLE_HTML_TAG = "title";
private static final String NO_BASE_URI = "";
private static final Map<String, List<String>> EMPTY_METADATA = ImmutableMap.of();
+ private static final MimeType TEXT_HTML = MimeType.of("text/html");
+ private static final MimeType TEXT_PLAIN = MimeType.of("text/plain");
@Override
- public ParsedContent extractContent(InputStream inputStream, String contentType) throws Exception {
+ public ParsedContent extractContent(InputStream inputStream, ContentType contentType) throws Exception {
if (inputStream == null || contentType == null) {
return ParsedContent.empty();
}
- if (contentType.equals("text/html")) {
+ if (contentType.mimeType().equals(TEXT_HTML)) {
return parseHtmlContent(inputStream);
}
- if (contentType.equals("text/plain")) {
+ if (contentType.mimeType().equals(TEXT_PLAIN)) {
return parsePlainTextContent(inputStream);
}
return ParsedContent.empty();
diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearches.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearches.java
index 690fcbf..583ec25 100644
--- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearches.java
+++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearches.java
@@ -267,8 +267,7 @@ public class MessageSearches implements Iterable<SimpleMessageSearchIndex.Search
return textExtractor
.extractContent(
rawData,
- // todo we likely want only the media type here
- attachment.getType().asString())
+ attachment.getType())
.getTextualContent()
.stream();
} catch (Exception e) {
diff --git a/mailbox/store/src/test/java/org/apache/james/mailbox/store/extractor/DefaultTextExtractorTest.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/extractor/DefaultTextExtractorTest.java
index 49ae39e..aaac5b9 100644
--- a/mailbox/store/src/test/java/org/apache/james/mailbox/store/extractor/DefaultTextExtractorTest.java
+++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/extractor/DefaultTextExtractorTest.java
@@ -24,6 +24,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import java.io.InputStream;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.model.ContentType;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -39,18 +40,27 @@ class DefaultTextExtractorTest {
void textTest() throws Exception {
InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/Text.txt");
assertThat(inputStream).isNotNull();
- assertThat(textExtractor.extractContent(inputStream, "text/plain")
+ assertThat(textExtractor.extractContent(inputStream, ContentType.of("text/plain"))
.getTextualContent())
.contains("This is some awesome text text.\n\n");
}
@Test
+ void extractContentShouldTakeIntoAccountCharset() throws Exception {
+ InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/simple-text-iso-8859-1.txt");
+ assertThat(inputStream).isNotNull();
+ assertThat(textExtractor.extractContent(inputStream, ContentType.of("text/plain; charset=ISO-8859-1"))
+ .getTextualContent())
+ .contains("\"é\" This text is not UTF-8 \"à\"");
+ }
+
+ @Test
void textMicrosoftWorldTest() throws Exception {
InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/writter.docx");
assertThat(inputStream).isNotNull();
assertThat(textExtractor.extractContent(
inputStream,
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document")
+ ContentType.of("application/vnd.openxmlformats-officedocument.wordprocessingml.document"))
.getTextualContent())
.isEmpty();
}
diff --git a/mailbox/store/src/test/java/org/apache/james/mailbox/store/extractor/JsoupTextExtractorTest.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/extractor/JsoupTextExtractorTest.java
index cfdd362..00d00f9 100644
--- a/mailbox/store/src/test/java/org/apache/james/mailbox/store/extractor/JsoupTextExtractorTest.java
+++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/extractor/JsoupTextExtractorTest.java
@@ -27,12 +27,13 @@ import java.nio.charset.StandardCharsets;
import org.apache.james.mailbox.extractor.ParsedContent;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.model.ContentType;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
class JsoupTextExtractorTest {
- private static final String TEXT_HTML_CONTENT_TYPE = "text/html";
+ private static final ContentType TEXT_HTML_CONTENT_TYPE = ContentType.of("text/html");
private static final String HTML_TEXT_CONTENT = "HTML pages can include a lot of null '\0' character. But still expecting the content can be parsed." +
"Jsoup 1.12.1 thinks a file containing more than 10 null characters can be a binary file";
private static final String NULL_CHARACTERS = "\0\0\0\0\0\0\0\0\0\0";
@@ -57,7 +58,15 @@ class JsoupTextExtractorTest {
void extractContentShouldHandlePlainText() throws Exception {
InputStream inputStream = new ByteArrayInputStream("myText".getBytes(StandardCharsets.UTF_8));
- assertThat(textExtractor.extractContent(inputStream, "text/plain").getTextualContent())
+ assertThat(textExtractor.extractContent(inputStream, ContentType.of("text/plain")).getTextualContent())
+ .contains("myText");
+ }
+
+ @Test
+ void extractContentShouldHandlePlainTextWithCharset() throws Exception {
+ InputStream inputStream = new ByteArrayInputStream("myText".getBytes(StandardCharsets.UTF_8));
+
+ assertThat(textExtractor.extractContent(inputStream, ContentType.of("text/plain; charset=utf-8")).getTextualContent())
.contains("myText");
}
@@ -65,7 +74,7 @@ class JsoupTextExtractorTest {
void extractContentShouldHandleArbitraryTextMediaType() throws Exception {
InputStream inputStream = new ByteArrayInputStream("myText".getBytes(StandardCharsets.UTF_8));
- assertThat(textExtractor.extractContent(inputStream, "text/arbitrary").getTextualContent())
+ assertThat(textExtractor.extractContent(inputStream, ContentType.of("text/arbitrary")).getTextualContent())
.isEmpty();
}
@@ -93,6 +102,24 @@ class JsoupTextExtractorTest {
.contains(HTML_TEXT_CONTENT));
}
+ @Test
+ void extractContentShouldTakeIntoAccountCharsetWhenPlainText() throws Exception {
+ InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/simple-text-iso-8859-1.txt");
+ assertThat(inputStream).isNotNull();
+ assertThat(textExtractor.extractContent(inputStream, ContentType.of("text/plain; charset=ISO-8859-1"))
+ .getTextualContent())
+ .contains("\"é\" This text is not UTF-8 \"à\"");
+ }
+
+ @Test
+ void extractContentTakeIntoAccountCharsetWhenHTML() throws Exception {
+ InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/html-iso-8859-1.html");
+ assertThat(inputStream).isNotNull();
+ assertThat(textExtractor.extractContent(inputStream, ContentType.of("text/html; charset=ISO-8859-1"))
+ .getTextualContent())
+ .contains("\"é\" this is a simple HTML text \"à\"");
+ }
+
private InputStream textContentWithManyNullCharacters() {
return new ByteArrayInputStream(FULL_CONTENT.getBytes(StandardCharsets.UTF_8));
}
diff --git a/mailbox/api/src/main/java/org/apache/james/mailbox/extractor/TextExtractor.java b/mailbox/store/src/test/resources/documents/html-iso-8859-1.html
similarity index 69%
copy from mailbox/api/src/main/java/org/apache/james/mailbox/extractor/TextExtractor.java
copy to mailbox/store/src/test/resources/documents/html-iso-8859-1.html
index b33d853..ad6c89d 100644
--- a/mailbox/api/src/main/java/org/apache/james/mailbox/extractor/TextExtractor.java
+++ b/mailbox/store/src/test/resources/documents/html-iso-8859-1.html
@@ -1,4 +1,4 @@
-/****************************************************************
+<!--
* Licensed to the Apache Software Foundation (ASF) under one *
* or more contributor license agreements. See the NOTICE file *
* distributed with this work for additional information *
@@ -15,14 +15,17 @@
* KIND, either express or implied. See the License for the *
* specific language governing permissions and limitations *
* under the License. *
- ****************************************************************/
+-->
+<!doctype html>
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office">
+ <head>
+ <![endif]-->
+ <meta charset="UTF-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <title>*|MC:SUBJECT|*</title>
-package org.apache.james.mailbox.extractor;
-
-import java.io.InputStream;
-
-public interface TextExtractor {
-
- ParsedContent extractContent(InputStream inputStream, String contentType) throws Exception;
-
-}
+ <body>
+ <p>"�" this is a simple HTML text "�"</p>
+ </body>
+</html>
diff --git a/mailbox/store/src/test/resources/documents/simple-text-iso-8859-1.txt b/mailbox/store/src/test/resources/documents/simple-text-iso-8859-1.txt
new file mode 100644
index 0000000..3be593c
--- /dev/null
+++ b/mailbox/store/src/test/resources/documents/simple-text-iso-8859-1.txt
@@ -0,0 +1 @@
+"�" This text is not UTF-8 "�"
\ No newline at end of file
diff --git a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
index 9feaf55..30cd94f 100644
--- a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
+++ b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
@@ -29,6 +29,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.io.IOUtils;
import org.apache.james.mailbox.extractor.ParsedContent;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.model.ContentType;
import org.apache.james.metrics.api.GaugeRegistry;
import org.apache.james.metrics.api.Metric;
import org.apache.james.metrics.api.MetricFactory;
@@ -108,7 +109,7 @@ public class CachingTextExtractor implements TextExtractor {
}
@Override
- public ParsedContent extractContent(InputStream inputStream, String contentType) throws Exception {
+ public ParsedContent extractContent(InputStream inputStream, ContentType contentType) throws Exception {
byte[] bytes = IOUtils.toByteArray(inputStream);
String key = Hashing.sha256().hashBytes(bytes).toString();
@@ -119,7 +120,7 @@ public class CachingTextExtractor implements TextExtractor {
}
}
- private ParsedContent retrieveAndUpdateWeight(byte[] bytes, String contentType) throws Exception {
+ private ParsedContent retrieveAndUpdateWeight(byte[] bytes, ContentType contentType) throws Exception {
ParsedContent parsedContent = underlying.extractContent(new ByteArrayInputStream(bytes), contentType);
weightMetric.add(computeWeight(parsedContent));
return parsedContent;
diff --git a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/ContentTypeFilteringTextExtractor.java b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/ContentTypeFilteringTextExtractor.java
index c648610..d846b75 100644
--- a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/ContentTypeFilteringTextExtractor.java
+++ b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/ContentTypeFilteringTextExtractor.java
@@ -23,28 +23,30 @@ import java.io.InputStream;
import org.apache.james.mailbox.extractor.ParsedContent;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.model.ContentType;
+import org.apache.james.mailbox.model.ContentType.MimeType;
import com.google.common.collect.ImmutableSet;
public class ContentTypeFilteringTextExtractor implements TextExtractor {
private final TextExtractor textExtractor;
- private final ImmutableSet<String> contentTypeBlacklist;
+ private final ImmutableSet<MimeType> contentTypeBlacklist;
- public ContentTypeFilteringTextExtractor(TextExtractor textExtractor, ImmutableSet<String> contentTypeBlacklist) {
+ public ContentTypeFilteringTextExtractor(TextExtractor textExtractor, ImmutableSet<MimeType> contentTypeBlacklist) {
this.textExtractor = textExtractor;
this.contentTypeBlacklist = contentTypeBlacklist;
}
@Override
- public ParsedContent extractContent(InputStream inputStream, String contentType) throws Exception {
- if (isBlacklisted(contentType)) {
+ public ParsedContent extractContent(InputStream inputStream, ContentType contentType) throws Exception {
+ if (isBlacklisted(contentType.mimeType())) {
return ParsedContent.empty();
}
return textExtractor.extractContent(inputStream, contentType);
}
- private boolean isBlacklisted(String contentType) {
+ private boolean isBlacklisted(MimeType contentType) {
return contentTypeBlacklist.contains(contentType);
}
diff --git a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaConfiguration.java b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaConfiguration.java
index e95a8e8..590461c 100644
--- a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaConfiguration.java
+++ b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaConfiguration.java
@@ -25,6 +25,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import org.apache.james.mailbox.model.ContentType.MimeType;
import org.apache.james.util.Port;
import com.google.common.base.Preconditions;
@@ -41,7 +42,7 @@ public class TikaConfiguration {
private Optional<Integer> timeoutInMillis;
private Optional<Duration> cacheEvictionPeriod;
private Optional<Long> cacheWeightInBytes;
- private ImmutableSet.Builder<String> contentTypeBlacklist;
+ private ImmutableSet.Builder<MimeType> contentTypeBlacklist;
private Builder() {
isEnabled = Optional.empty();
@@ -140,7 +141,7 @@ public class TikaConfiguration {
return this;
}
- public Builder contentTypeBlacklist(Set<String> contentTypeBlacklist) {
+ public Builder contentTypeBlacklist(Set<MimeType> contentTypeBlacklist) {
Preconditions.checkNotNull(contentTypeBlacklist);
this.contentTypeBlacklist.addAll(contentTypeBlacklist);
return this;
@@ -179,9 +180,9 @@ public class TikaConfiguration {
private final int timeoutInMillis;
private final Duration cacheEvictionPeriod;
private final long cacheWeightInBytes;
- private final ImmutableSet<String> contentTypeBlacklist;
+ private final ImmutableSet<MimeType> contentTypeBlacklist;
- private TikaConfiguration(boolean enabled, boolean cacheEnabled, String host, int port, int timeoutInMillis, Duration cacheEvictionPeriod, long cacheWeightInBytes, ImmutableSet<String> contentTypeBlacklist) {
+ private TikaConfiguration(boolean enabled, boolean cacheEnabled, String host, int port, int timeoutInMillis, Duration cacheEvictionPeriod, long cacheWeightInBytes, ImmutableSet<MimeType> contentTypeBlacklist) {
this.enabled = enabled;
this.cacheEnabled = cacheEnabled;
this.host = host;
@@ -220,7 +221,7 @@ public class TikaConfiguration {
return cacheWeightInBytes;
}
- public ImmutableSet<String> getContentTypeBlacklist() {
+ public ImmutableSet<MimeType> getContentTypeBlacklist() {
return contentTypeBlacklist;
}
diff --git a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaHttpClient.java b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaHttpClient.java
index 9e490db..ceae8ff 100644
--- a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaHttpClient.java
+++ b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaHttpClient.java
@@ -21,7 +21,9 @@ package org.apache.james.mailbox.tika;
import java.io.InputStream;
import java.util.Optional;
+import org.apache.james.mailbox.model.ContentType;
+
public interface TikaHttpClient {
- Optional<InputStream> recursiveMetaDataAsJson(InputStream inputStream, String contentType);
+ Optional<InputStream> recursiveMetaDataAsJson(InputStream inputStream, ContentType contentType);
}
diff --git a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaHttpClientImpl.java b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaHttpClientImpl.java
index 402eb6b..5f00ca9 100644
--- a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaHttpClientImpl.java
+++ b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaHttpClientImpl.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
+import java.nio.charset.Charset;
import java.util.Optional;
import org.apache.http.client.fluent.Request;
@@ -52,12 +53,16 @@ public class TikaHttpClientImpl implements TikaHttpClient {
}
@Override
- public Optional<InputStream> recursiveMetaDataAsJson(InputStream inputStream, String contentType) {
+ public Optional<InputStream> recursiveMetaDataAsJson(InputStream inputStream, org.apache.james.mailbox.model.ContentType contentType) {
try {
+ ContentType httpContentType = ContentType.create(contentType.mimeType().asString(),
+ contentType.charset()
+ .map(Charset::name)
+ .orElse(null));
return Optional.ofNullable(
Request.Put(recursiveMetaData)
.socketTimeout(tikaConfiguration.getTimeoutInMillis())
- .bodyStream(inputStream, ContentType.create(contentType))
+ .bodyStream(inputStream, httpContentType)
.execute()
.returnContent()
.asStream());
diff --git a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaTextExtractor.java b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaTextExtractor.java
index d12bfa4..b20da09 100644
--- a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaTextExtractor.java
+++ b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaTextExtractor.java
@@ -33,6 +33,7 @@ import javax.inject.Inject;
import org.apache.commons.lang3.StringUtils;
import org.apache.james.mailbox.extractor.ParsedContent;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.model.ContentType;
import org.apache.james.mailbox.store.extractor.JsoupTextExtractor;
import org.apache.james.metrics.api.MetricFactory;
@@ -56,6 +57,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
public class TikaTextExtractor implements TextExtractor {
+ private static final ContentType.MediaType TEXT = ContentType.MediaType.of("text");
private final MetricFactory metricFactory;
private final TikaHttpClient tikaHttpClient;
@@ -79,8 +81,8 @@ public class TikaTextExtractor implements TextExtractor {
}
@Override
- public ParsedContent extractContent(InputStream inputStream, String contentType) throws Exception {
- if (contentType.startsWith("text/")) {
+ public ParsedContent extractContent(InputStream inputStream, ContentType contentType) throws Exception {
+ if (contentType.mediaType().equals(TEXT)) {
return jsoupTextExtractor.extractContent(inputStream, contentType);
}
return metricFactory.runPublishingTimerMetric("tikaTextExtraction", Throwing.supplier(
@@ -88,7 +90,7 @@ public class TikaTextExtractor implements TextExtractor {
.sneakyThrow());
}
- public ParsedContent performContentExtraction(InputStream inputStream, String contentType) throws IOException {
+ public ParsedContent performContentExtraction(InputStream inputStream, ContentType contentType) throws IOException {
ContentAndMetadata contentAndMetadata = convert(tikaHttpClient.recursiveMetaDataAsJson(inputStream, contentType));
return new ParsedContent(contentAndMetadata.getContent(), contentAndMetadata.getMetadata());
}
diff --git a/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java b/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
index 791a7ed..6722733 100644
--- a/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
+++ b/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
@@ -42,6 +42,7 @@ import java.util.stream.IntStream;
import org.apache.james.mailbox.extractor.ParsedContent;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.model.ContentType;
import org.apache.james.metrics.api.NoopGaugeRegistry;
import org.apache.james.metrics.tests.RecordingMetricFactory;
import org.apache.james.util.concurrency.ConcurrentTestRunner;
@@ -62,7 +63,7 @@ public class CachingTextExtractorTest {
i -> new ByteArrayInputStream(String.format("content%d", i).getBytes(StandardCharsets.UTF_8));
private static final Supplier<InputStream> INPUT_STREAM = () -> STREAM_GENERATOR.apply(1);
private static final long CACHE_LIMIT_10_MiB = 10 * 1024 * 1024;
- private static final String CONTENT_TYPE = "application/bytes";
+ private static final ContentType CONTENT_TYPE = ContentType.of("application/bytes");
private CachingTextExtractor textExtractor;
private TextExtractor wrappedTextExtractor;
diff --git a/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/ContentTypeFilteringTextExtractorTest.java b/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/ContentTypeFilteringTextExtractorTest.java
index ebea68c..89825a2 100644
--- a/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/ContentTypeFilteringTextExtractorTest.java
+++ b/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/ContentTypeFilteringTextExtractorTest.java
@@ -31,6 +31,8 @@ import java.nio.charset.StandardCharsets;
import org.apache.commons.io.IOUtils;
import org.apache.james.mailbox.extractor.ParsedContent;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.model.ContentType;
+import org.apache.james.mailbox.model.ContentType.MimeType;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mock;
@@ -52,10 +54,22 @@ class ContentTypeFilteringTextExtractorTest {
void extractContentReturnEmptyWithContentTypeInBlacklist() throws Exception {
ContentTypeFilteringTextExtractor contentTypeFilteringTextExtractor =
new ContentTypeFilteringTextExtractor(textExtractor,
- ImmutableSet.of("application/ics", "application/zip"));
+ ImmutableSet.of(MimeType.of("application/ics"), MimeType.of("application/zip")));
assertThat(contentTypeFilteringTextExtractor
- .extractContent(IOUtils.toInputStream("", StandardCharsets.UTF_8), "application/ics"))
+ .extractContent(IOUtils.toInputStream("", StandardCharsets.UTF_8), ContentType.of("application/ics")))
+ .isEqualTo(ParsedContent.empty());
+ verifyNoMoreInteractions(textExtractor);
+ }
+
+ @Test
+ void extractContentShouldIgnoreContentTypeParameters() throws Exception {
+ ContentTypeFilteringTextExtractor contentTypeFilteringTextExtractor =
+ new ContentTypeFilteringTextExtractor(textExtractor,
+ ImmutableSet.of(MimeType.of("application/ics"), MimeType.of("application/zip")));
+
+ assertThat(contentTypeFilteringTextExtractor
+ .extractContent(IOUtils.toInputStream("", StandardCharsets.UTF_8), ContentType.of("application/ics; charset=utf-8")))
.isEqualTo(ParsedContent.empty());
verifyNoMoreInteractions(textExtractor);
}
@@ -65,8 +79,8 @@ class ContentTypeFilteringTextExtractorTest {
InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/Text.txt");
ContentTypeFilteringTextExtractor contentTypeFilteringTextExtractor =
new ContentTypeFilteringTextExtractor(textExtractor,
- ImmutableSet.of("application/ics", "application/zip"));
- contentTypeFilteringTextExtractor.extractContent(inputStream, "text/plain");
+ ImmutableSet.of(MimeType.of("application/ics"), MimeType.of("application/zip")));
+ contentTypeFilteringTextExtractor.extractContent(inputStream, ContentType.of("text/plain"));
verify(textExtractor, times(1)).extractContent(any(), any());
}
diff --git a/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/TikaTextExtractorTest.java b/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/TikaTextExtractorTest.java
index 4ecde3d..e47cee2 100644
--- a/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/TikaTextExtractorTest.java
+++ b/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/TikaTextExtractorTest.java
@@ -31,6 +31,7 @@ import java.util.Optional;
import org.apache.commons.io.IOUtils;
import org.apache.james.mailbox.extractor.ParsedContent;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.model.ContentType;
import org.apache.james.mailbox.tika.TikaTextExtractor.ContentAndMetadataDeserializer;
import org.apache.james.metrics.tests.RecordingMetricFactory;
import org.junit.jupiter.api.BeforeEach;
@@ -59,7 +60,8 @@ class TikaTextExtractorTest {
@Test
void textualContentShouldReturnEmptyWhenInputStreamIsEmpty() throws Exception {
- assertThat(textExtractor.extractContent(IOUtils.toInputStream("", StandardCharsets.UTF_8), "text/plain").getTextualContent())
+ assertThat(textExtractor.extractContent(IOUtils.toInputStream("", StandardCharsets.UTF_8), ContentType.of("text/plain"))
+ .getTextualContent())
.contains("");
}
@@ -67,7 +69,7 @@ class TikaTextExtractorTest {
void textTest() throws Exception {
InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/Text.txt");
assertThat(inputStream).isNotNull();
- assertThat(textExtractor.extractContent(inputStream, "text/plain").getTextualContent())
+ assertThat(textExtractor.extractContent(inputStream, ContentType.of("text/plain")).getTextualContent())
.isPresent()
.asString()
.contains("This is some awesome text text.");
@@ -77,7 +79,9 @@ class TikaTextExtractorTest {
void textMicrosoftWorldTest() throws Exception {
InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/writter.docx");
assertThat(inputStream).isNotNull();
- assertThat(textExtractor.extractContent(inputStream, "application/vnd.openxmlformats-officedocument.wordprocessingml.document").getTextualContent())
+ assertThat(textExtractor.extractContent(inputStream,
+ ContentType.of("application/vnd.openxmlformats-officedocument.wordprocessingml.document"))
+ .getTextualContent())
.isPresent()
.asString()
.contains("This is an awesome document on libroffice writter !");
@@ -87,7 +91,8 @@ class TikaTextExtractorTest {
void textOdtTest() throws Exception {
InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/writter.odt");
assertThat(inputStream).isNotNull();
- assertThat(textExtractor.extractContent(inputStream, "application/vnd.oasis.opendocument.text").getTextualContent())
+ assertThat(textExtractor.extractContent(inputStream, ContentType.of("application/vnd.oasis.opendocument.text"))
+ .getTextualContent())
.isPresent()
.asString()
.contains("This is an awesome document on libroffice writter !");
@@ -97,7 +102,8 @@ class TikaTextExtractorTest {
void documentWithBadDeclaredMetadataShouldBeWellHandled() throws Exception {
InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/fake.txt");
assertThat(inputStream).isNotNull();
- assertThat(textExtractor.extractContent(inputStream, "application/vnd.oasis.opendocument.text").getTextualContent())
+ assertThat(textExtractor.extractContent(inputStream, ContentType.of("application/vnd.oasis.opendocument.text"))
+ .getTextualContent())
.isPresent()
.asString()
.contains("This is an awesome document on libroffice writter !");
@@ -107,7 +113,7 @@ class TikaTextExtractorTest {
void slidePowerPointTest() throws Exception {
InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/slides.pptx");
assertThat(inputStream).isNotNull();
- assertThat(textExtractor.extractContent(inputStream, "application/vnd.openxmlformats-officedocument.presentationml.presentation").getTextualContent())
+ assertThat(textExtractor.extractContent(inputStream, ContentType.of("application/vnd.openxmlformats-officedocument.presentationml.presentation")).getTextualContent())
.isPresent()
.asString()
.contains("James is awesome")
@@ -118,7 +124,8 @@ class TikaTextExtractorTest {
void slideOdpTest() throws Exception {
InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/slides.odp");
assertThat(inputStream).isNotNull();
- assertThat(textExtractor.extractContent(inputStream, "application/vnd.oasis.opendocument.presentation").getTextualContent())
+ assertThat(textExtractor.extractContent(inputStream, ContentType.of("application/vnd.oasis.opendocument.presentation"))
+ .getTextualContent())
.isPresent()
.asString()
.contains("James is awesome")
@@ -129,7 +136,8 @@ class TikaTextExtractorTest {
void pdfTest() throws Exception {
InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/PDF.pdf");
assertThat(inputStream).isNotNull();
- assertThat(textExtractor.extractContent(inputStream, "application/pdf").getTextualContent())
+ assertThat(textExtractor.extractContent(inputStream, ContentType.of("application/pdf"))
+ .getTextualContent())
.isPresent()
.asString()
.contains("This is an awesome document on libroffice writter !");
@@ -139,7 +147,8 @@ class TikaTextExtractorTest {
void odsTest() throws Exception {
InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/calc.ods");
assertThat(inputStream).isNotNull();
- assertThat(textExtractor.extractContent(inputStream, "application/vnd.oasis.opendocument.spreadsheet").getTextualContent())
+ assertThat(textExtractor.extractContent(inputStream, ContentType.of("application/vnd.oasis.opendocument.spreadsheet"))
+ .getTextualContent())
.isPresent()
.asString()
.contains("This is an aesome LibreOffice document !");
@@ -149,7 +158,8 @@ class TikaTextExtractorTest {
void excelTest() throws Exception {
InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/calc.xlsx");
assertThat(inputStream).isNotNull();
- assertThat(textExtractor.extractContent(inputStream, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet").getTextualContent())
+ assertThat(textExtractor.extractContent(inputStream, ContentType.of("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"))
+ .getTextualContent())
.isPresent()
.asString()
.contains("Feuille1")
@@ -165,7 +175,7 @@ class TikaTextExtractorTest {
.getBytes(StandardCharsets.UTF_8))));
InputStream inputStream = null;
- textExtractor.extractContent(inputStream, "text/plain");
+ textExtractor.extractContent(inputStream, ContentType.of("text/plain"));
}
@Test
@@ -178,7 +188,8 @@ class TikaTextExtractorTest {
.getBytes(StandardCharsets.UTF_8))));
InputStream inputStream = new ByteArrayInputStream("toto".getBytes(StandardCharsets.UTF_8));
- ParsedContent parsedContent = textExtractor.extractContent(inputStream, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
+ ParsedContent parsedContent = textExtractor.extractContent(inputStream,
+ ContentType.of("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"));
assertThat(parsedContent.getTextualContent()).contains(expectedExtractedContent);
}
@@ -192,7 +203,8 @@ class TikaTextExtractorTest {
InputStream inputStream = new ByteArrayInputStream("toto".getBytes(StandardCharsets.UTF_8));
- assertThatThrownBy(() -> textExtractor.extractContent(inputStream, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"))
+ assertThatThrownBy(() -> textExtractor.extractContent(inputStream,
+ ContentType.of("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")))
.isInstanceOf(IllegalStateException.class)
.hasMessage("The element should be a Json object");
}
diff --git a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaConfigurationReader.java b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaConfigurationReader.java
index ae7bc0e..341f838 100644
--- a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaConfigurationReader.java
+++ b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaConfigurationReader.java
@@ -25,6 +25,7 @@ import java.util.Optional;
import java.util.Set;
import org.apache.commons.configuration2.Configuration;
+import org.apache.james.mailbox.model.ContentType.MimeType;
import org.apache.james.mailbox.tika.TikaConfiguration;
import org.apache.james.util.DurationParser;
import org.apache.james.util.Size;
@@ -69,9 +70,10 @@ public class TikaConfigurationReader {
.map(Throwing.function(Size::parse))
.map(Size::asBytes);
- Set<String> contentTypeBlacklist = StreamUtils
+ Set<MimeType> contentTypeBlacklist = StreamUtils
.ofNullable(configuration.getStringArray(TIKA_CONTENT_TYPE_BLACKLIST))
.map(String::trim)
+ .map(MimeType::of)
.collect(ImmutableSet.toImmutableSet());
return TikaConfiguration.builder()
diff --git a/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/TikaConfigurationReaderTest.java b/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/TikaConfigurationReaderTest.java
index 03b8a99..aefa878 100644
--- a/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/TikaConfigurationReaderTest.java
+++ b/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/TikaConfigurationReaderTest.java
@@ -26,6 +26,7 @@ import java.time.Duration;
import org.apache.commons.configuration2.PropertiesConfiguration;
import org.apache.commons.configuration2.convert.DefaultListDelimiterHandler;
+import org.apache.james.mailbox.model.ContentType.MimeType;
import org.apache.james.mailbox.tika.TikaConfiguration;
import org.junit.Test;
@@ -272,7 +273,7 @@ public class TikaConfigurationReaderTest {
.port(889)
.timeoutInMillis(500)
.cacheWeightInBytes(1520000)
- .contentTypeBlacklist(ImmutableSet.of("application/ics", "application/zip"))
+ .contentTypeBlacklist(ImmutableSet.of(MimeType.of("application/ics"), MimeType.of("application/zip")))
.build());
}
@@ -297,7 +298,7 @@ public class TikaConfigurationReaderTest {
.port(889)
.timeoutInMillis(500)
.cacheWeightInBytes(1520000)
- .contentTypeBlacklist(ImmutableSet.of("application/ics", "application/zip"))
+ .contentTypeBlacklist(ImmutableSet.of(MimeType.of("application/ics"), MimeType.of("application/zip")))
.build());
}
diff --git a/server/protocols/jmap-draft/src/main/java/org/apache/james/jmap/draft/methods/MIMEMessageConverter.java b/server/protocols/jmap-draft/src/main/java/org/apache/james/jmap/draft/methods/MIMEMessageConverter.java
index a7c9dc7..64f9dd3 100644
--- a/server/protocols/jmap-draft/src/main/java/org/apache/james/jmap/draft/methods/MIMEMessageConverter.java
+++ b/server/protocols/jmap-draft/src/main/java/org/apache/james/jmap/draft/methods/MIMEMessageConverter.java
@@ -328,9 +328,7 @@ public class MIMEMessageConverter {
}
private ContentTypeField contentTypeField(MessageAttachmentMetadata att) {
- // todo mailbox pojo should be able to expose itself as a mime4j object
- String type = att.getAttachment().getType().asString();
- ContentTypeField typeAsField = Fields.contentType(type);
+ ContentTypeField typeAsField = att.getAttachment().getType().asMime4J();
if (att.getName().isPresent()) {
return Fields.contentType(typeAsField.getMimeType(),
ImmutableMap.<String, String>builder()
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org