You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by th...@apache.org on 2014/04/04 11:21:10 UTC

svn commit: r1584574 - in /tika/trunk/tika-parsers/src: main/java/org/apache/tika/parser/mbox/OutlookPSTParser.java test/java/org/apache/tika/parser/mbox/OutlookPSTParserTest.java

Author: thaichat04
Date: Fri Apr  4 09:21:10 2014
New Revision: 1584574

URL: http://svn.apache.org/r1584574
Log:
TIKA-623 - Extract each mail as attachment

Modified:
    tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/mbox/OutlookPSTParser.java
    tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/mbox/OutlookPSTParserTest.java

Modified: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/mbox/OutlookPSTParser.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/mbox/OutlookPSTParser.java?rev=1584574&r1=1584573&r2=1584574&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/mbox/OutlookPSTParser.java (original)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/mbox/OutlookPSTParser.java Fri Apr  4 09:21:10 2014
@@ -19,6 +19,7 @@ package org.apache.tika.parser.mbox;
 import static java.lang.String.valueOf;
 import static java.util.Collections.singleton;
 
+import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
@@ -30,6 +31,7 @@ import org.apache.tika.extractor.Parsing
 import org.apache.tika.io.TemporaryResources;
 import org.apache.tika.io.TikaInputStream;
 import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
 import org.apache.tika.mime.MediaType;
 import org.apache.tika.parser.AbstractParser;
 import org.apache.tika.parser.ParseContext;
@@ -50,129 +52,146 @@ import com.pff.PSTMessage;
  */
 public class OutlookPSTParser extends AbstractParser {
 
+  private static final long serialVersionUID = 620998217748364063L;
 
-    private static final long serialVersionUID = 620998217748364063L;
+  private static final MediaType MS_OUTLOOK_PST_MIMETYPE = MediaType.application("vnd.ms-outlook-pst");
+  private static final Set<MediaType> SUPPORTED_TYPES = singleton(MS_OUTLOOK_PST_MIMETYPE);
 
-    private static final MediaType MS_OUTLOOK_PST_MIMETYPE = MediaType.application("vnd.ms-outlook-pst");
-    private static final Set<MediaType> SUPPORTED_TYPES = singleton(MS_OUTLOOK_PST_MIMETYPE);
-
-    public Set<MediaType> getSupportedTypes(ParseContext context) {
-        return SUPPORTED_TYPES;
-    }
-
-    public void parse(InputStream stream, ContentHandler handler,
-            Metadata metadata, ParseContext context) throws IOException,
-            SAXException, TikaException {
-
-        // Use the delegate parser to parse the contained document
-        EmbeddedDocumentExtractor embeddedExtractor = context.get(
-                EmbeddedDocumentExtractor.class,
-                new ParsingEmbeddedDocumentExtractor(context));
-
-        metadata.set(Metadata.CONTENT_TYPE, MS_OUTLOOK_PST_MIMETYPE.toString());
-
-        XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
-        xhtml.startDocument();
-
-        TikaInputStream in = TikaInputStream.get(stream);
-        try {
-            PSTFile pstFile = new PSTFile(in.getFile().getPath());
-            metadata.set(Metadata.CONTENT_LENGTH, valueOf(pstFile.getFileHandle().length()));
-            boolean isValid = pstFile.getFileHandle().getFD().valid();
-            metadata.set("isValid", valueOf(isValid));
-            if (isValid) {
-                parseFolder(xhtml, pstFile.getRootFolder(), embeddedExtractor);
-            }
-        } catch (Exception e) {
-            throw new TikaException(e.getMessage());
+  public Set<MediaType> getSupportedTypes(ParseContext context) {
+    return SUPPORTED_TYPES;
+  }
+
+  public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context)
+      throws IOException, SAXException, TikaException {
+
+    // Use the delegate parser to parse the contained document
+    EmbeddedDocumentExtractor embeddedExtractor = context.get(EmbeddedDocumentExtractor.class,
+        new ParsingEmbeddedDocumentExtractor(context));
+
+    metadata.set(Metadata.CONTENT_TYPE, MS_OUTLOOK_PST_MIMETYPE.toString());
+
+    XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
+    xhtml.startDocument();
+
+    TikaInputStream in = TikaInputStream.get(stream);
+    try {
+      PSTFile pstFile = new PSTFile(in.getFile().getPath());
+      metadata.set(Metadata.CONTENT_LENGTH, valueOf(pstFile.getFileHandle().length()));
+      boolean isValid = pstFile.getFileHandle().getFD().valid();
+      metadata.set("isValid", valueOf(isValid));
+      if (isValid) {
+        parseFolder(xhtml, pstFile.getRootFolder(), embeddedExtractor);
+      }
+    } catch (Exception e) {
+      throw new TikaException(e.getMessage(), e);
+    }
+
+    xhtml.endDocument();
+  }
+
+  private void parseFolder(XHTMLContentHandler handler, PSTFolder pstFolder, EmbeddedDocumentExtractor embeddedExtractor)
+      throws Exception {
+    if (pstFolder.getContentCount() > 0) {
+      PSTMessage pstMail = (PSTMessage) pstFolder.getNextChild();
+      while (pstMail != null) {
+        AttributesImpl attributes = new AttributesImpl();
+        attributes.addAttribute("", "class", "class", "CDATA", "embedded");
+        attributes.addAttribute("", "id", "id", "CDATA", pstMail.getInternetMessageId());
+        handler.startElement("div", attributes);
+        handler.element("h1", pstMail.getSubject());
+
+        parserMailItem(handler, pstMail, embeddedExtractor);
+        parseMailAttachments(handler, pstMail, embeddedExtractor);
+
+        handler.endElement("div");
+
+        pstMail = (PSTMessage) pstFolder.getNextChild();
+      }
+    }
+
+    if (pstFolder.hasSubfolders()) {
+      for (PSTFolder pstSubFolder : pstFolder.getSubFolders()) {
+        handler.startElement("div", createAttribute("class", "email-folder"));
+        handler.element("h1", pstSubFolder.getDisplayName());
+        parseFolder(handler, pstSubFolder, embeddedExtractor);
+        handler.endElement("div");
+      }
+    }
+  }
+
+  private void parserMailItem(XHTMLContentHandler handler, PSTMessage pstMail, EmbeddedDocumentExtractor embeddedExtractor) throws SAXException, IOException {
+    Metadata mailMetadata = new Metadata();
+    mailMetadata.set(Metadata.RESOURCE_NAME_KEY, pstMail.getInternetMessageId());
+    mailMetadata.set(Metadata.EMBEDDED_RELATIONSHIP_ID, pstMail.getInternetMessageId());
+    mailMetadata.set(TikaCoreProperties.IDENTIFIER, pstMail.getInternetMessageId());
+    mailMetadata.set(TikaCoreProperties.TITLE, pstMail.getSubject());
+    mailMetadata.set(Metadata.MESSAGE_FROM, pstMail.getSenderName());
+    mailMetadata.set(TikaCoreProperties.CREATOR, pstMail.getSenderName());
+    mailMetadata.set(TikaCoreProperties.CREATED, pstMail.getCreationTime());
+    mailMetadata.set(TikaCoreProperties.MODIFIED, pstMail.getLastModificationTime());
+    mailMetadata.set(TikaCoreProperties.COMMENTS, pstMail.getComment());
+    mailMetadata.set("descriptorNodeId", valueOf(pstMail.getDescriptorNodeId()));
+    mailMetadata.set("senderEmailAddress", pstMail.getSenderEmailAddress());
+    mailMetadata.set("recipients", pstMail.getRecipientsString());
+    mailMetadata.set("displayTo", pstMail.getDisplayTo());
+    mailMetadata.set("displayCC", pstMail.getDisplayCC());
+    mailMetadata.set("displayBCC", pstMail.getDisplayBCC());
+    mailMetadata.set("importance", valueOf(pstMail.getImportance()));
+    mailMetadata.set("priority", valueOf(pstMail.getPriority()));
+    mailMetadata.set("flagged", valueOf(pstMail.isFlagged()));
+
+    byte[] mailContent = pstMail.getBody().getBytes();
+    embeddedExtractor.parseEmbedded(new ByteArrayInputStream(mailContent), handler, mailMetadata, true);
+  }
+
+
+  private static AttributesImpl createAttribute(String attName, String attValue) {
+    AttributesImpl attributes = new AttributesImpl();
+    attributes.addAttribute("", attName, attName, "CDATA", attValue);
+    return attributes;
+  }
+
+  private void parseMailAttachments(XHTMLContentHandler xhtml, PSTMessage email, EmbeddedDocumentExtractor embeddedExtractor)
+      throws TikaException {
+    int numberOfAttachments = email.getNumberOfAttachments();
+    for (int i = 0; i < numberOfAttachments; i++) {
+      File tempFile = null;
+      try {
+        PSTAttachment attach = email.getAttachment(i);
+
+        // Get the filename; both long and short filenames can be used for attachments
+        String filename = attach.getLongFilename();
+        if (filename.isEmpty()) {
+          filename = attach.getFilename();
         }
 
-        xhtml.endDocument();
-    }
-
-    private static void parseFolder(XHTMLContentHandler xhtml, PSTFolder pstFolder, EmbeddedDocumentExtractor embeddedExtractor) throws Exception {
-        if (pstFolder.getContentCount() > 0) {
-            PSTMessage pstMail = (PSTMessage) pstFolder.getNextChild();
-            while (pstMail != null) {
-                xhtml.startElement("div", createAttribute("class", "email-entry"));
-                xhtml.element("h1", pstMail.getInternetMessageId());
-                createMetadata(xhtml, "subject", pstMail.getSubject());
-                createMetadata(xhtml, "internetMessageId", valueOf(pstMail.getInternetMessageId()));
-                createMetadata(xhtml, "descriptorNodeId", valueOf(pstMail.getDescriptorNodeId()));
-                createMetadata(xhtml, "lastModificationTime", valueOf(pstMail.getLastModificationTime().getTime()));
-                createMetadata(xhtml, "senderName", pstMail.getSenderName());
-                createMetadata(xhtml, "senderEmailAddress", pstMail.getSenderEmailAddress());
-                createMetadata(xhtml, "recipients", pstMail.getRecipientsString());
-
-                xhtml.element("p", pstMail.getBody());
-                parseAttachments(xhtml, pstMail, embeddedExtractor);
-
-                xhtml.endElement("div");
-                pstMail = (PSTMessage) pstFolder.getNextChild();
-            }
-        }
+        xhtml.element("p", filename);
 
-        if (pstFolder.hasSubfolders()) {
-            for (PSTFolder pstSubFolder : pstFolder.getSubFolders()) {
-                xhtml.startElement("div", createAttribute("class", "email-folder"));
-                xhtml.element("h1", pstSubFolder.getDisplayName());
-                parseFolder(xhtml, pstSubFolder, embeddedExtractor);
-                xhtml.endElement("div");
-            }
+        Metadata attachMeta = new Metadata();
+        attachMeta.set(Metadata.RESOURCE_NAME_KEY, filename);
+        attachMeta.set(Metadata.EMBEDDED_RELATIONSHIP_ID, filename);
+        AttributesImpl attributes = new AttributesImpl();
+        attributes.addAttribute("", "class", "class", "CDATA", "embedded");
+        attributes.addAttribute("", "id", "id", "CDATA", filename);
+        xhtml.startElement("div", attributes);
+        if (embeddedExtractor.shouldParseEmbedded(attachMeta)) {
+          TemporaryResources tmp = new TemporaryResources();
+          try {
+            TikaInputStream tis = TikaInputStream.get(attach.getFileInputStream(), tmp);
+            embeddedExtractor.parseEmbedded(tis, xhtml, attachMeta, true);
+          } finally {
+            tmp.dispose();
+          }
         }
-    }
-
-    private static void createMetadata(XHTMLContentHandler xhtml, String metaName, String metaValue) throws SAXException {
-        xhtml.startElement("meta", createAttribute(metaName, metaValue));
-        xhtml.endElement("meta");
-    }
+        xhtml.endElement("div");
 
-    private static AttributesImpl createAttribute(String attName, String attValue) {
-        AttributesImpl attributes  = new AttributesImpl();
-        attributes.addAttribute("", attName, attName, "CDATA", attValue);
-        return attributes;
-    }
-
-    private static void parseAttachments(XHTMLContentHandler xhtml, PSTMessage email, EmbeddedDocumentExtractor embeddedExtractor) throws TikaException {
-        int numberOfAttachments = email.getNumberOfAttachments();
-        for (int i = 0; i < numberOfAttachments; i++) {
-            File tempFile = null;
-            try {
-                PSTAttachment attach = email.getAttachment(i);
-
-                // Get the filename; both long and short filenames can be used for attachments
-                String filename = attach.getLongFilename();
-                if (filename.isEmpty()) {
-                    filename = attach.getFilename();
-                }
-
-                xhtml.element("p", filename);
-
-                Metadata attachMeta = new Metadata();
-                attachMeta.set(Metadata.RESOURCE_NAME_KEY, filename);
-                attachMeta.set(Metadata.EMBEDDED_RELATIONSHIP_ID, filename);
-                AttributesImpl attributes = new AttributesImpl();
-                attributes.addAttribute("", "class", "class", "CDATA", "embedded");
-                attributes.addAttribute("", "id", "id", "CDATA", filename);
-                xhtml.startElement("div", attributes);
-                if (embeddedExtractor.shouldParseEmbedded(attachMeta)) {
-                    TemporaryResources tmp = new TemporaryResources();
-                    try {
-                        TikaInputStream tis = TikaInputStream.get(attach.getFileInputStream(), tmp);
-                        embeddedExtractor.parseEmbedded(tis, xhtml, attachMeta, true);
-                    } finally {
-                        tmp.dispose();
-                    }
-                }
-                xhtml.endElement("div");
-
-            } catch (Exception e) {
-                throw new TikaException("Unable to unpack document stream", e);
-            } finally {
-                if (tempFile != null)
-                    tempFile.delete();
-            }
-        }
+      } catch (Exception e) {
+        throw new TikaException("Unable to unpack document stream", e);
+      } finally {
+        if (tempFile != null)
+          tempFile.delete();
+      }
     }
+  }
 
 }

Modified: tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/mbox/OutlookPSTParserTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/mbox/OutlookPSTParserTest.java?rev=1584574&r1=1584573&r2=1584574&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/mbox/OutlookPSTParserTest.java (original)
+++ tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/mbox/OutlookPSTParserTest.java Fri Apr  4 09:21:10 2014
@@ -16,14 +16,28 @@
  */
 package org.apache.tika.parser.mbox;
 
+import static junit.framework.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.tika.TikaTest;
+import org.apache.tika.extractor.EmbeddedDocumentExtractor;
+import org.apache.tika.extractor.ParsingEmbeddedDocumentExtractor;
 import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
 import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.ParseContext;
 import org.apache.tika.parser.Parser;
+import org.apache.tika.sax.ToHTMLContentHandler;
 import org.junit.Test;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
 
 public class OutlookPSTParserTest extends TikaTest {
 
@@ -36,19 +50,61 @@ public class OutlookPSTParserTest extend
 
   @Test
   public void testParse() throws Exception {
-    XMLResult result = getXML(getResourceAsStream("/test-documents/testPST.pst"), new OutlookPSTParser(), new Metadata());
-    String xml = result.xml;
+    OutlookPSTParser pstParser = new OutlookPSTParser();
+    Metadata metadata = new Metadata();
+    ContentHandler handler = new ToHTMLContentHandler();
+
+    ParseContext context = new ParseContext();
+    EmbeddedTrackingExtrator trackingExtrator = new EmbeddedTrackingExtrator(context);
+    context.set(EmbeddedDocumentExtractor.class, trackingExtrator);
+    context.set(Parser.class, new AutoDetectParser());
+
+    pstParser.parse(getResourceAsStream("/test-documents/testPST.pst"), handler, metadata, context);
+
+    String output = handler.toString();
+
+    assertFalse(output.isEmpty());
+    assertTrue(output.contains("<meta name=\"Content-Length\" content=\"271360\">"));
+    assertTrue(output.contains("<meta name=\"Content-Type\" content=\"application/vnd.ms-outlook-pst\">"));
+
+    assertTrue(output.contains("<body><div class=\"email-folder\"><h1>"));
+    assertTrue(output.contains("<div class=\"embedded\" id=\"&lt;530D9CAC.5080901@gmail.com&gt;\"><h1>Re: Feature Generators</h1>"));
+    assertTrue(output.contains("<div class=\"embedded\" id=\"&lt;1393363252.28814.YahooMailNeo@web140906.mail.bf1.yahoo.com&gt;\"><h1>Re: init tokenizer fails: \"Bad type in putfield/putstatic\"</h1>"));
+    assertTrue(output.contains("Gary Murphy commented on TIKA-1250:"));
 
-    assertFalse(xml.isEmpty());
-    assertTrue(xml.contains("<meta name=\"Content-Length\" content=\"271360\" />"));
-    assertTrue(xml.contains("<meta name=\"Content-Type\" content=\"application/vnd.ms-outlook-pst\" />"));
-
-    assertTrue(xml.contains("<body><div class=\"email-folder\"><h1>"));
-    assertTrue(xml.contains("<div class=\"email-entry\"><h1>&lt;530D9CAC.5080901@gmail.com&gt;</h1>"));
-    assertTrue(xml.contains("<div class=\"email-entry\"><h1>&lt;1393363252.28814.YahooMailNeo@web140906.mail.bf1.yahoo.com&gt;</h1>"));
-    assertTrue(xml.contains("Gary Murphy commented on TIKA-1250:"));
+    assertTrue(output.contains("<div class=\"email-folder\"><h1>Racine (pour la recherche)</h1>"));
 
-    assertTrue(xml.contains("<div class=\"email-folder\"><h1>Racine (pour la recherche)</h1>"));
+
+    List<Metadata> metaList = trackingExtrator.trackingMetadata;
+    assertEquals(6, metaList.size());
+
+    Metadata firstMail = metaList.get(0);
+    assertEquals("Jörn Kottmann", firstMail.get(TikaCoreProperties.CREATOR));
+    assertEquals("Re: Feature Generators", firstMail.get(TikaCoreProperties.TITLE));
+    assertEquals("kottmann@gmail.com", firstMail.get("senderEmailAddress"));
+    assertEquals("users@opennlp.apache.org", firstMail.get("displayTo"));
+    assertEquals("", firstMail.get("displayCC"));
+    assertEquals("", firstMail.get("displayBCC"));
   }
 
+
+  private class EmbeddedTrackingExtrator extends ParsingEmbeddedDocumentExtractor {
+    List<Metadata> trackingMetadata = new ArrayList<Metadata>();
+
+    public EmbeddedTrackingExtrator(ParseContext context) {
+      super(context);
+    }
+
+    @Override
+    public boolean shouldParseEmbedded(Metadata metadata) {
+      return true;
+    }
+
+    @Override
+    public void parseEmbedded(InputStream stream, ContentHandler handler, Metadata metadata, boolean outputHtml) throws SAXException, IOException {
+      this.trackingMetadata.add(metadata);
+      super.parseEmbedded(stream, handler, metadata, outputHtml);
+    }
+
+  }
 }