svn commit: r1104444 - in /tika/trunk/tika-parsers/src: main/java/org/apache/tika/parser/mail/RFC822Parser.java test/java/org/apache/tika/parser/mail/RFC822ParserTest.java

[ju...@apache.org]

Author: jukka
Date: Tue May 17 18:07:13 2011
New Revision: 1104444

URL: http://svn.apache.org/viewvc?rev=1104444&view=rev
Log:
TIKA-640: RFC822Parser should configure Mime4j not to fail reading mails containing more than 1000 chars in one headers text (even if folded)

Allow a custom MimeEntityConfig to be passed as a part of the parsing context.

Also capture Mime- and other IOExceptions that are not directly caused by IO problems from the underlying stream.

Modified:
    tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/mail/RFC822Parser.java
    tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java

Modified: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/mail/RFC822Parser.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/mail/RFC822Parser.java?rev=1104444&r1=1104443&r2=1104444&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/mail/RFC822Parser.java (original)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/mail/RFC822Parser.java Tue May 17 18:07:13 2011
@@ -25,6 +25,7 @@ import org.apache.james.mime4j.MimeExcep
 import org.apache.james.mime4j.parser.MimeEntityConfig;
 import org.apache.james.mime4j.parser.MimeStreamParser;
 import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TaggedInputStream;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.mime.MediaType;
 import org.apache.tika.parser.AbstractParser;
@@ -36,7 +37,10 @@ import org.xml.sax.SAXException;
 /**
  * Uses apache-mime4j to parse emails. Each part is treated with the
  * corresponding parser and displayed within elements.
- * 
+ * <p>
+ * A {@link MimeEntityConfig} object can be passed in the parsing context
+ * to better control the parsing process.
+ *
  * @author jnioche@digitalpebble.com
  **/
 public class RFC822Parser extends AbstractParser {
@@ -51,16 +55,23 @@ public class RFC822Parser extends Abstra
     public void parse(InputStream stream, ContentHandler handler,
             Metadata metadata, ParseContext context) throws IOException,
             SAXException, TikaException {
+        // Get the mime4j configuration, or use a default one
         MimeEntityConfig config = new MimeEntityConfig();
-        config.setMaxLineLen(10000); //this is max length of any individual header
+        config.setMaxLineLen(10000); // max length of any individual header
+        config = context.get(MimeEntityConfig.class, config);
+
         MimeStreamParser parser = new MimeStreamParser(config);
         XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
 
         MailContentHandler mch = new MailContentHandler(xhtml, metadata);
         parser.setContentHandler(mch);
         parser.setContentDecoding(true);
+        TaggedInputStream tagged = TaggedInputStream.get(stream);
         try {
-            parser.parse(stream);
+            parser.parse(tagged);
+        } catch (IOException e) {
+            tagged.throwIfCauseOf(e);
+            throw new TikaException("Failed to parse an email message", e);
         } catch (MimeException e) {
             // Unwrap the exception in case it was not thrown by mime4j
             Throwable cause = e.getCause();

Modified: tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java?rev=1104444&r1=1104443&r2=1104444&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java (original)
+++ tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java Tue May 17 18:07:13 2011
@@ -23,10 +23,13 @@ import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 
+import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 
 import junit.framework.TestCase;
 
+import org.apache.james.mime4j.parser.MimeEntityConfig;
+import org.apache.tika.exception.TikaException;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.parser.ParseContext;
 import org.apache.tika.parser.Parser;
@@ -165,6 +168,38 @@ public class RFC822ParserTest extends Te
        assertEquals("Air Permit Programs | Air & Radiation | US EPA", metadata.get(Metadata.SUBJECT));
     }
 
+    /**
+     * Test for TIKA-640, increase header max beyond 10k bytes
+     */
+    public void testLongHeader() throws Exception {
+        StringBuilder inputBuilder = new StringBuilder();
+        for (int i = 0; i < 200; ++i) {
+            inputBuilder.append( //len > 50
+                    "really really really really really really long name ");
+        }
+        String name = inputBuilder.toString();
+        byte[] data = ("From: " + name + "\r\n\r\n").getBytes("US-ASCII");
+
+        Parser parser = new RFC822Parser();
+        ContentHandler handler = new DefaultHandler();
+        Metadata metadata = new Metadata();
+        ParseContext context = new ParseContext();
+
+        try {
+            parser.parse(
+                    new ByteArrayInputStream(data), handler, metadata, context);
+            fail();
+        } catch (TikaException expected) {
+        }
+
+        MimeEntityConfig config = new MimeEntityConfig();
+        config.setMaxLineLen(-1);
+        context.set(MimeEntityConfig.class, config);
+        parser.parse(
+                new ByteArrayInputStream(data), handler, metadata, context);
+        assertEquals(name, metadata.get(Metadata.AUTHOR));
+    }
+
     private static InputStream getStream(String name) {
         return Thread.currentThread().getContextClassLoader()
                 .getResourceAsStream(name);