You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by th...@apache.org on 2014/04/04 11:21:10 UTC
svn commit: r1584574 - in /tika/trunk/tika-parsers/src:
main/java/org/apache/tika/parser/mbox/OutlookPSTParser.java
test/java/org/apache/tika/parser/mbox/OutlookPSTParserTest.java
Author: thaichat04
Date: Fri Apr 4 09:21:10 2014
New Revision: 1584574
URL: http://svn.apache.org/r1584574
Log:
TIKA-623 - Extract each mail as attachment
Modified:
tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/mbox/OutlookPSTParser.java
tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/mbox/OutlookPSTParserTest.java
Modified: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/mbox/OutlookPSTParser.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/mbox/OutlookPSTParser.java?rev=1584574&r1=1584573&r2=1584574&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/mbox/OutlookPSTParser.java (original)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/mbox/OutlookPSTParser.java Fri Apr 4 09:21:10 2014
@@ -19,6 +19,7 @@ package org.apache.tika.parser.mbox;
import static java.lang.String.valueOf;
import static java.util.Collections.singleton;
+import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -30,6 +31,7 @@ import org.apache.tika.extractor.Parsing
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.AbstractParser;
import org.apache.tika.parser.ParseContext;
@@ -50,129 +52,146 @@ import com.pff.PSTMessage;
*/
public class OutlookPSTParser extends AbstractParser {
+ private static final long serialVersionUID = 620998217748364063L;
- private static final long serialVersionUID = 620998217748364063L;
+ private static final MediaType MS_OUTLOOK_PST_MIMETYPE = MediaType.application("vnd.ms-outlook-pst");
+ private static final Set<MediaType> SUPPORTED_TYPES = singleton(MS_OUTLOOK_PST_MIMETYPE);
- private static final MediaType MS_OUTLOOK_PST_MIMETYPE = MediaType.application("vnd.ms-outlook-pst");
- private static final Set<MediaType> SUPPORTED_TYPES = singleton(MS_OUTLOOK_PST_MIMETYPE);
-
- public Set<MediaType> getSupportedTypes(ParseContext context) {
- return SUPPORTED_TYPES;
- }
-
- public void parse(InputStream stream, ContentHandler handler,
- Metadata metadata, ParseContext context) throws IOException,
- SAXException, TikaException {
-
- // Use the delegate parser to parse the contained document
- EmbeddedDocumentExtractor embeddedExtractor = context.get(
- EmbeddedDocumentExtractor.class,
- new ParsingEmbeddedDocumentExtractor(context));
-
- metadata.set(Metadata.CONTENT_TYPE, MS_OUTLOOK_PST_MIMETYPE.toString());
-
- XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
- xhtml.startDocument();
-
- TikaInputStream in = TikaInputStream.get(stream);
- try {
- PSTFile pstFile = new PSTFile(in.getFile().getPath());
- metadata.set(Metadata.CONTENT_LENGTH, valueOf(pstFile.getFileHandle().length()));
- boolean isValid = pstFile.getFileHandle().getFD().valid();
- metadata.set("isValid", valueOf(isValid));
- if (isValid) {
- parseFolder(xhtml, pstFile.getRootFolder(), embeddedExtractor);
- }
- } catch (Exception e) {
- throw new TikaException(e.getMessage());
+ public Set<MediaType> getSupportedTypes(ParseContext context) {
+ return SUPPORTED_TYPES;
+ }
+
+ public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context)
+ throws IOException, SAXException, TikaException {
+
+ // Use the delegate parser to parse the contained document
+ EmbeddedDocumentExtractor embeddedExtractor = context.get(EmbeddedDocumentExtractor.class,
+ new ParsingEmbeddedDocumentExtractor(context));
+
+ metadata.set(Metadata.CONTENT_TYPE, MS_OUTLOOK_PST_MIMETYPE.toString());
+
+ XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
+ xhtml.startDocument();
+
+ TikaInputStream in = TikaInputStream.get(stream);
+ try {
+ PSTFile pstFile = new PSTFile(in.getFile().getPath());
+ metadata.set(Metadata.CONTENT_LENGTH, valueOf(pstFile.getFileHandle().length()));
+ boolean isValid = pstFile.getFileHandle().getFD().valid();
+ metadata.set("isValid", valueOf(isValid));
+ if (isValid) {
+ parseFolder(xhtml, pstFile.getRootFolder(), embeddedExtractor);
+ }
+ } catch (Exception e) {
+ throw new TikaException(e.getMessage(), e);
+ }
+
+ xhtml.endDocument();
+ }
+
+ private void parseFolder(XHTMLContentHandler handler, PSTFolder pstFolder, EmbeddedDocumentExtractor embeddedExtractor)
+ throws Exception {
+ if (pstFolder.getContentCount() > 0) {
+ PSTMessage pstMail = (PSTMessage) pstFolder.getNextChild();
+ while (pstMail != null) {
+ AttributesImpl attributes = new AttributesImpl();
+ attributes.addAttribute("", "class", "class", "CDATA", "embedded");
+ attributes.addAttribute("", "id", "id", "CDATA", pstMail.getInternetMessageId());
+ handler.startElement("div", attributes);
+ handler.element("h1", pstMail.getSubject());
+
+ parserMailItem(handler, pstMail, embeddedExtractor);
+ parseMailAttachments(handler, pstMail, embeddedExtractor);
+
+ handler.endElement("div");
+
+ pstMail = (PSTMessage) pstFolder.getNextChild();
+ }
+ }
+
+ if (pstFolder.hasSubfolders()) {
+ for (PSTFolder pstSubFolder : pstFolder.getSubFolders()) {
+ handler.startElement("div", createAttribute("class", "email-folder"));
+ handler.element("h1", pstSubFolder.getDisplayName());
+ parseFolder(handler, pstSubFolder, embeddedExtractor);
+ handler.endElement("div");
+ }
+ }
+ }
+
+ private void parserMailItem(XHTMLContentHandler handler, PSTMessage pstMail, EmbeddedDocumentExtractor embeddedExtractor) throws SAXException, IOException {
+ Metadata mailMetadata = new Metadata();
+ mailMetadata.set(Metadata.RESOURCE_NAME_KEY, pstMail.getInternetMessageId());
+ mailMetadata.set(Metadata.EMBEDDED_RELATIONSHIP_ID, pstMail.getInternetMessageId());
+ mailMetadata.set(TikaCoreProperties.IDENTIFIER, pstMail.getInternetMessageId());
+ mailMetadata.set(TikaCoreProperties.TITLE, pstMail.getSubject());
+ mailMetadata.set(Metadata.MESSAGE_FROM, pstMail.getSenderName());
+ mailMetadata.set(TikaCoreProperties.CREATOR, pstMail.getSenderName());
+ mailMetadata.set(TikaCoreProperties.CREATED, pstMail.getCreationTime());
+ mailMetadata.set(TikaCoreProperties.MODIFIED, pstMail.getLastModificationTime());
+ mailMetadata.set(TikaCoreProperties.COMMENTS, pstMail.getComment());
+ mailMetadata.set("descriptorNodeId", valueOf(pstMail.getDescriptorNodeId()));
+ mailMetadata.set("senderEmailAddress", pstMail.getSenderEmailAddress());
+ mailMetadata.set("recipients", pstMail.getRecipientsString());
+ mailMetadata.set("displayTo", pstMail.getDisplayTo());
+ mailMetadata.set("displayCC", pstMail.getDisplayCC());
+ mailMetadata.set("displayBCC", pstMail.getDisplayBCC());
+ mailMetadata.set("importance", valueOf(pstMail.getImportance()));
+ mailMetadata.set("priority", valueOf(pstMail.getPriority()));
+ mailMetadata.set("flagged", valueOf(pstMail.isFlagged()));
+
+ byte[] mailContent = pstMail.getBody().getBytes();
+ embeddedExtractor.parseEmbedded(new ByteArrayInputStream(mailContent), handler, mailMetadata, true);
+ }
+
+
+ private static AttributesImpl createAttribute(String attName, String attValue) {
+ AttributesImpl attributes = new AttributesImpl();
+ attributes.addAttribute("", attName, attName, "CDATA", attValue);
+ return attributes;
+ }
+
+ private void parseMailAttachments(XHTMLContentHandler xhtml, PSTMessage email, EmbeddedDocumentExtractor embeddedExtractor)
+ throws TikaException {
+ int numberOfAttachments = email.getNumberOfAttachments();
+ for (int i = 0; i < numberOfAttachments; i++) {
+ File tempFile = null;
+ try {
+ PSTAttachment attach = email.getAttachment(i);
+
+ // Get the filename; both long and short filenames can be used for attachments
+ String filename = attach.getLongFilename();
+ if (filename.isEmpty()) {
+ filename = attach.getFilename();
}
- xhtml.endDocument();
- }
-
- private static void parseFolder(XHTMLContentHandler xhtml, PSTFolder pstFolder, EmbeddedDocumentExtractor embeddedExtractor) throws Exception {
- if (pstFolder.getContentCount() > 0) {
- PSTMessage pstMail = (PSTMessage) pstFolder.getNextChild();
- while (pstMail != null) {
- xhtml.startElement("div", createAttribute("class", "email-entry"));
- xhtml.element("h1", pstMail.getInternetMessageId());
- createMetadata(xhtml, "subject", pstMail.getSubject());
- createMetadata(xhtml, "internetMessageId", valueOf(pstMail.getInternetMessageId()));
- createMetadata(xhtml, "descriptorNodeId", valueOf(pstMail.getDescriptorNodeId()));
- createMetadata(xhtml, "lastModificationTime", valueOf(pstMail.getLastModificationTime().getTime()));
- createMetadata(xhtml, "senderName", pstMail.getSenderName());
- createMetadata(xhtml, "senderEmailAddress", pstMail.getSenderEmailAddress());
- createMetadata(xhtml, "recipients", pstMail.getRecipientsString());
-
- xhtml.element("p", pstMail.getBody());
- parseAttachments(xhtml, pstMail, embeddedExtractor);
-
- xhtml.endElement("div");
- pstMail = (PSTMessage) pstFolder.getNextChild();
- }
- }
+ xhtml.element("p", filename);
- if (pstFolder.hasSubfolders()) {
- for (PSTFolder pstSubFolder : pstFolder.getSubFolders()) {
- xhtml.startElement("div", createAttribute("class", "email-folder"));
- xhtml.element("h1", pstSubFolder.getDisplayName());
- parseFolder(xhtml, pstSubFolder, embeddedExtractor);
- xhtml.endElement("div");
- }
+ Metadata attachMeta = new Metadata();
+ attachMeta.set(Metadata.RESOURCE_NAME_KEY, filename);
+ attachMeta.set(Metadata.EMBEDDED_RELATIONSHIP_ID, filename);
+ AttributesImpl attributes = new AttributesImpl();
+ attributes.addAttribute("", "class", "class", "CDATA", "embedded");
+ attributes.addAttribute("", "id", "id", "CDATA", filename);
+ xhtml.startElement("div", attributes);
+ if (embeddedExtractor.shouldParseEmbedded(attachMeta)) {
+ TemporaryResources tmp = new TemporaryResources();
+ try {
+ TikaInputStream tis = TikaInputStream.get(attach.getFileInputStream(), tmp);
+ embeddedExtractor.parseEmbedded(tis, xhtml, attachMeta, true);
+ } finally {
+ tmp.dispose();
+ }
}
- }
-
- private static void createMetadata(XHTMLContentHandler xhtml, String metaName, String metaValue) throws SAXException {
- xhtml.startElement("meta", createAttribute(metaName, metaValue));
- xhtml.endElement("meta");
- }
+ xhtml.endElement("div");
- private static AttributesImpl createAttribute(String attName, String attValue) {
- AttributesImpl attributes = new AttributesImpl();
- attributes.addAttribute("", attName, attName, "CDATA", attValue);
- return attributes;
- }
-
- private static void parseAttachments(XHTMLContentHandler xhtml, PSTMessage email, EmbeddedDocumentExtractor embeddedExtractor) throws TikaException {
- int numberOfAttachments = email.getNumberOfAttachments();
- for (int i = 0; i < numberOfAttachments; i++) {
- File tempFile = null;
- try {
- PSTAttachment attach = email.getAttachment(i);
-
- // Get the filename; both long and short filenames can be used for attachments
- String filename = attach.getLongFilename();
- if (filename.isEmpty()) {
- filename = attach.getFilename();
- }
-
- xhtml.element("p", filename);
-
- Metadata attachMeta = new Metadata();
- attachMeta.set(Metadata.RESOURCE_NAME_KEY, filename);
- attachMeta.set(Metadata.EMBEDDED_RELATIONSHIP_ID, filename);
- AttributesImpl attributes = new AttributesImpl();
- attributes.addAttribute("", "class", "class", "CDATA", "embedded");
- attributes.addAttribute("", "id", "id", "CDATA", filename);
- xhtml.startElement("div", attributes);
- if (embeddedExtractor.shouldParseEmbedded(attachMeta)) {
- TemporaryResources tmp = new TemporaryResources();
- try {
- TikaInputStream tis = TikaInputStream.get(attach.getFileInputStream(), tmp);
- embeddedExtractor.parseEmbedded(tis, xhtml, attachMeta, true);
- } finally {
- tmp.dispose();
- }
- }
- xhtml.endElement("div");
-
- } catch (Exception e) {
- throw new TikaException("Unable to unpack document stream", e);
- } finally {
- if (tempFile != null)
- tempFile.delete();
- }
- }
+ } catch (Exception e) {
+ throw new TikaException("Unable to unpack document stream", e);
+ } finally {
+ if (tempFile != null)
+ tempFile.delete();
+ }
}
+ }
}
Modified: tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/mbox/OutlookPSTParserTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/mbox/OutlookPSTParserTest.java?rev=1584574&r1=1584573&r2=1584574&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/mbox/OutlookPSTParserTest.java (original)
+++ tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/mbox/OutlookPSTParserTest.java Fri Apr 4 09:21:10 2014
@@ -16,14 +16,28 @@
*/
package org.apache.tika.parser.mbox;
+import static junit.framework.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.tika.TikaTest;
+import org.apache.tika.extractor.EmbeddedDocumentExtractor;
+import org.apache.tika.extractor.ParsingEmbeddedDocumentExtractor;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
+import org.apache.tika.sax.ToHTMLContentHandler;
import org.junit.Test;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
public class OutlookPSTParserTest extends TikaTest {
@@ -36,19 +50,61 @@ public class OutlookPSTParserTest extend
@Test
public void testParse() throws Exception {
- XMLResult result = getXML(getResourceAsStream("/test-documents/testPST.pst"), new OutlookPSTParser(), new Metadata());
- String xml = result.xml;
+ OutlookPSTParser pstParser = new OutlookPSTParser();
+ Metadata metadata = new Metadata();
+ ContentHandler handler = new ToHTMLContentHandler();
+
+ ParseContext context = new ParseContext();
+ EmbeddedTrackingExtrator trackingExtrator = new EmbeddedTrackingExtrator(context);
+ context.set(EmbeddedDocumentExtractor.class, trackingExtrator);
+ context.set(Parser.class, new AutoDetectParser());
+
+ pstParser.parse(getResourceAsStream("/test-documents/testPST.pst"), handler, metadata, context);
+
+ String output = handler.toString();
+
+ assertFalse(output.isEmpty());
+ assertTrue(output.contains("<meta name=\"Content-Length\" content=\"271360\">"));
+ assertTrue(output.contains("<meta name=\"Content-Type\" content=\"application/vnd.ms-outlook-pst\">"));
+
+ assertTrue(output.contains("<body><div class=\"email-folder\"><h1>"));
+ assertTrue(output.contains("<div class=\"embedded\" id=\"<530D9CAC.5080901@gmail.com>\"><h1>Re: Feature Generators</h1>"));
+ assertTrue(output.contains("<div class=\"embedded\" id=\"<1393363252.28814.YahooMailNeo@web140906.mail.bf1.yahoo.com>\"><h1>Re: init tokenizer fails: \"Bad type in putfield/putstatic\"</h1>"));
+ assertTrue(output.contains("Gary Murphy commented on TIKA-1250:"));
- assertFalse(xml.isEmpty());
- assertTrue(xml.contains("<meta name=\"Content-Length\" content=\"271360\" />"));
- assertTrue(xml.contains("<meta name=\"Content-Type\" content=\"application/vnd.ms-outlook-pst\" />"));
-
- assertTrue(xml.contains("<body><div class=\"email-folder\"><h1>"));
- assertTrue(xml.contains("<div class=\"email-entry\"><h1><530D9CAC.5080901@gmail.com></h1>"));
- assertTrue(xml.contains("<div class=\"email-entry\"><h1><1393363252.28814.YahooMailNeo@web140906.mail.bf1.yahoo.com></h1>"));
- assertTrue(xml.contains("Gary Murphy commented on TIKA-1250:"));
+ assertTrue(output.contains("<div class=\"email-folder\"><h1>Racine (pour la recherche)</h1>"));
- assertTrue(xml.contains("<div class=\"email-folder\"><h1>Racine (pour la recherche)</h1>"));
+
+ List<Metadata> metaList = trackingExtrator.trackingMetadata;
+ assertEquals(6, metaList.size());
+
+ Metadata firstMail = metaList.get(0);
+ assertEquals("Jörn Kottmann", firstMail.get(TikaCoreProperties.CREATOR));
+ assertEquals("Re: Feature Generators", firstMail.get(TikaCoreProperties.TITLE));
+ assertEquals("kottmann@gmail.com", firstMail.get("senderEmailAddress"));
+ assertEquals("users@opennlp.apache.org", firstMail.get("displayTo"));
+ assertEquals("", firstMail.get("displayCC"));
+ assertEquals("", firstMail.get("displayBCC"));
}
+
+ private class EmbeddedTrackingExtrator extends ParsingEmbeddedDocumentExtractor {
+ List<Metadata> trackingMetadata = new ArrayList<Metadata>();
+
+ public EmbeddedTrackingExtrator(ParseContext context) {
+ super(context);
+ }
+
+ @Override
+ public boolean shouldParseEmbedded(Metadata metadata) {
+ return true;
+ }
+
+ @Override
+ public void parseEmbedded(InputStream stream, ContentHandler handler, Metadata metadata, boolean outputHtml) throws SAXException, IOException {
+ this.trackingMetadata.add(metadata);
+ super.parseEmbedded(stream, handler, metadata, outputHtml);
+ }
+
+ }
}