You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2016/10/17 18:42:27 UTC
tika git commit: TIKA-2122: Extract all headers from MSG/RFC822
Repository: tika
Updated Branches:
refs/heads/2.x 1e55953bc -> 30e03de89
TIKA-2122: Extract all headers from MSG/RFC822
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/30e03de8
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/30e03de8
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/30e03de8
Branch: refs/heads/2.x
Commit: 30e03de89fd4b21cb91917c72aec12eede761be3
Parents: 1e55953
Author: tballison <ta...@mitre.org>
Authored: Mon Oct 17 14:30:33 2016 -0400
Committer: tballison <ta...@mitre.org>
Committed: Mon Oct 17 14:30:33 2016 -0400
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../java/org/apache/tika/metadata/Message.java | 6 ++
.../src/test/java/org/apache/tika/TikaTest.java | 8 ++
.../tika-parser-office-bundle/pom.xml | 3 +-
tika-parser-modules/pom.xml | 1 +
.../tika-parser-office-module/pom.xml | 5 ++
.../tika/parser/microsoft/OutlookExtractor.java | 85 +++++++++++++++++++-
.../parser/microsoft/OutlookParserTest.java | 15 ++++
.../tika-parser-web-module/pom.xml | 5 +-
.../tika/parser/mail/MailContentHandler.java | 5 ++
.../tika/parser/mail/RFC822ParserTest.java | 1 +
11 files changed, 130 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 226d9c1..48180ed 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -17,6 +17,8 @@ Release 2.0 - ???
Release 1.14 - ???
+ * Extract all headers from MSG/RFC822 (TIKA-2122).
+
* Upgrade metadata extractor to 2.9.1 (TIKA-2113).
* Extract PDF DocInfo metadata into separate keys to prevent
http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-core/src/main/java/org/apache/tika/metadata/Message.java
----------------------------------------------------------------------
diff --git a/tika-core/src/main/java/org/apache/tika/metadata/Message.java b/tika-core/src/main/java/org/apache/tika/metadata/Message.java
index ffb9413..dad3952 100644
--- a/tika-core/src/main/java/org/apache/tika/metadata/Message.java
+++ b/tika-core/src/main/java/org/apache/tika/metadata/Message.java
@@ -16,10 +16,16 @@
*/
package org.apache.tika.metadata;
+import org.apache.tika.Tika;
+
/**
* A collection of Message related property names.
*/
public interface Message {
+ String MESSAGE_PREFIX = "Message"+ Metadata.NAMESPACE_PREFIX_DELIMITER;
+
+ String MESSAGE_RAW_HEADER_PREFIX = MESSAGE_PREFIX+"Raw-Header"+Metadata.NAMESPACE_PREFIX_DELIMITER;
+
String MESSAGE_RECIPIENT_ADDRESS = "Message-Recipient-Address";
String MESSAGE_FROM = "Message-From";
http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-core/src/test/java/org/apache/tika/TikaTest.java
----------------------------------------------------------------------
diff --git a/tika-core/src/test/java/org/apache/tika/TikaTest.java b/tika-core/src/test/java/org/apache/tika/TikaTest.java
index 847eb79..cfed800 100644
--- a/tika-core/src/test/java/org/apache/tika/TikaTest.java
+++ b/tika-core/src/test/java/org/apache/tika/TikaTest.java
@@ -322,4 +322,12 @@ public abstract class TikaTest {
i++;
}
}
+
+ public static void debug(Metadata metadata) {
+ for (String n : metadata.names()) {
+ for (String v : metadata.getValues(n)) {
+ System.out.println(n + " : "+v);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-parser-bundles/tika-parser-office-bundle/pom.xml
----------------------------------------------------------------------
diff --git a/tika-parser-bundles/tika-parser-office-bundle/pom.xml b/tika-parser-bundles/tika-parser-office-bundle/pom.xml
index 1529c97..1ef8b05 100644
--- a/tika-parser-bundles/tika-parser-office-bundle/pom.xml
+++ b/tika-parser-bundles/tika-parser-office-bundle/pom.xml
@@ -69,9 +69,10 @@
poi;inline=true,
poi-scratchpad;inline=true,
poi-ooxml;inline=true,
- poi-ooxml-schemas;inline=true;
+ poi-ooxml-schemas;inline=true,
jackcess;inline=true,
jackcess-encrypt;inline=true,
+ apache-mime4j-core;inline=true,
java-libpst;inline=true,
curvesapi;inline=true,
xmlbeans;inline=true,
http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-parser-modules/pom.xml
----------------------------------------------------------------------
diff --git a/tika-parser-modules/pom.xml b/tika-parser-modules/pom.xml
index dc3b409..c99cd0f 100644
--- a/tika-parser-modules/pom.xml
+++ b/tika-parser-modules/pom.xml
@@ -43,6 +43,7 @@
<!-- used by POI, PDFBox and Jackcess ...try to sync -->
<bouncycastle.version>1.54</bouncycastle.version>
<commons.exec>1.3</commons.exec>
+ <mime4j.version>0.7.2</mime4j.version>
</properties>
<modules>
http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-parser-modules/tika-parser-office-module/pom.xml
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/pom.xml b/tika-parser-modules/tika-parser-office-module/pom.xml
index 9cd6c2f..9c50c31 100644
--- a/tika-parser-modules/tika-parser-office-module/pom.xml
+++ b/tika-parser-modules/tika-parser-office-module/pom.xml
@@ -91,6 +91,11 @@
<version>0.8.1</version>
</dependency>
<dependency>
+ <groupId>org.apache.james</groupId>
+ <artifactId>apache-mime4j-core</artifactId>
+ <version>${mime4j.version}</version>
+ </dependency>
+ <dependency>
<groupId>${project.groupId}</groupId>
<artifactId>tika-parser-package-module</artifactId>
<version>${project.version}</version>
http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
index d8d26aa..17d45d2 100644
--- a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
+++ b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
@@ -26,13 +26,17 @@ import java.nio.charset.IllegalCharsetNameException;
import java.nio.charset.UnsupportedCharsetException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.Date;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.james.mime4j.codec.DecodeMonitor;
+import org.apache.james.mime4j.codec.DecoderUtil;
import org.apache.poi.hmef.attribute.MAPIRtfAttribute;
import org.apache.poi.hsmf.MAPIMessage;
import org.apache.poi.hsmf.datatypes.AttachmentChunks;
@@ -68,6 +72,18 @@ import org.xml.sax.SAXException;
* Outlook Message Parser.
*/
public class OutlookExtractor extends AbstractPOIFSExtractor {
+
+
+ private static Pattern HEADER_KEY_PAT =
+ Pattern.compile("\\A([\\x21-\\x39\\x3B-\\x7E]+):(.*?)\\Z");
+ //this according to the spec; in practice, it is probably more likely
+ //that a "split field" fails to start with a space character than
+ //that a real header contains anything but [-_A-Za-z0-9].
+ //e.g.
+ //header: this header goes onto the next line
+ //<mailto:xyz@cnn.com...
+
+
private static final Metadata EMPTY_METADATA = new Metadata();
private final SimpleDateFormat dateFormat;
private final EncodingDetector htmlEncodingDetectorProxy;
@@ -124,8 +140,19 @@ public class OutlookExtractor extends AbstractPOIFSExtractor {
}
} catch (ChunkNotFoundException he) {
} // Will be fixed in POI 3.7 Final
+ try {
+ Map<String, String[]> headers = normalizeHeaders(msg.getHeaders());
+ for (Map.Entry<String, String[]> e : headers.entrySet()) {
+ String headerKey = e.getKey();
+ for (String headerValue : e.getValue()) {
+ metadata.add(Metadata.MESSAGE_RAW_HEADER_PREFIX+headerKey, headerValue);
+ }
+ }
+ } catch (ChunkNotFoundException e) {
+
+ }
- // Date - try two ways to find it
+ // Date - try two ways to find it
// First try via the proper chunk
if (msg.getMessageDate() != null) {
metadata.set(TikaCoreProperties.CREATED, msg.getMessageDate().getTime());
@@ -270,6 +297,62 @@ public class OutlookExtractor extends AbstractPOIFSExtractor {
}
}
+ //As of 3.15, POI currently returns header[] by splitting on /\r?\n/
+ //this rebuilds headers that are broken up over several lines
+ //this also decodes encoded headers.
+ private Map<String, String[]> normalizeHeaders(String[] rows) {
+ Map<String, String[]> ret = new LinkedHashMap<>();
+ if (rows == null) {
+ return ret;
+ }
+ StringBuilder sb = new StringBuilder();
+ Map<String, List<String>> headers = new LinkedHashMap();
+ Matcher headerKeyMatcher = HEADER_KEY_PAT.matcher("");
+ String lastKey = null;
+ int consec = 0;
+ for (String row : rows) {
+ headerKeyMatcher.reset(row);
+ if (headerKeyMatcher.find()) {
+ if (lastKey != null) {
+ List<String> vals = headers.get(lastKey);
+ vals = (vals == null) ? new ArrayList<String>() : vals;
+ vals.add(decodeHeader(sb.toString()));
+ headers.put(lastKey, vals);
+ }
+ //reset sb
+ sb.setLength(0);
+ lastKey = headerKeyMatcher.group(1).trim();
+ sb.append(headerKeyMatcher.group(2).trim());
+ consec = 0;
+ } else {
+ if (consec > 0) {
+ sb.append("\n");
+ }
+ sb.append(row);
+ }
+ consec++;
+ }
+
+ //make sure to add the last value
+ if (sb.length() > 0 && lastKey != null) {
+ List<String> vals = headers.get(lastKey);
+ vals = (vals == null) ? new ArrayList<String>() : vals;
+ vals.add(decodeHeader(sb.toString()));
+ headers.put(lastKey, vals);
+ }
+
+ //convert to array
+ for (Map.Entry<String, List<String>> e : headers.entrySet()) {
+ ret.put(e.getKey(), e.getValue().toArray(new String[e.getValue().size()]));
+ }
+ return ret;
+
+ }
+
+ private String decodeHeader(String header) {
+ return DecoderUtil.decodeEncodedWords(header, DecodeMonitor.SILENT);
+ }
+
private void header(XHTMLContentHandler xhtml, String key, String value)
throws SAXException {
if (value != null && value.length() > 0) {
http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java
index 8662e65..c15308f 100644
--- a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java
+++ b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java
@@ -26,6 +26,7 @@ import javax.xml.transform.sax.TransformerHandler;
import javax.xml.transform.stream.StreamResult;
import java.io.InputStream;
import java.io.StringWriter;
+import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -71,6 +72,12 @@ public class OutlookParserTest extends TikaTest {
"L'\u00C9quipe Microsoft Outlook Express",
metadata.get(Metadata.AUTHOR));
+ //ensure that "raw" header is correctly decoded
+ assertEquals(
+ "L'\u00C9quipe Microsoft Outlook Express <ms...@microsoft.com>",
+ metadata.get(Metadata.MESSAGE_RAW_HEADER_PREFIX+"From"));
+
+
// Stored as Thu, 5 Apr 2007 09:26:06 -0700
assertEquals(
"2007-04-05T16:26:06Z",
@@ -108,6 +115,14 @@ public class OutlookParserTest extends TikaTest {
Matcher matcher = pattern.matcher(content);
assertTrue(matcher.find());
assertFalse(matcher.find());
+
+ //test that last header is added
+ assertContains("29 Jan 2009 19:17:10.0163 (UTC) FILETIME=[2ED25E30:01C98246]",
+ Arrays.asList(metadata.getValues("Message:Raw-Header:X-OriginalArrivalTime")));
+ //confirm next line is added correctly
+ assertContains("from athena.apache.org (HELO athena.apache.org) (140.211.11.136)\n" +
+ " by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 29 Jan 2009 11:17:08 -0800",
+ Arrays.asList(metadata.getValues("Message:Raw-Header:Received")));
}
/**
http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-parser-modules/tika-parser-web-module/pom.xml
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-web-module/pom.xml b/tika-parser-modules/tika-parser-web-module/pom.xml
index ee9e24c..6f11184 100644
--- a/tika-parser-modules/tika-parser-web-module/pom.xml
+++ b/tika-parser-modules/tika-parser-web-module/pom.xml
@@ -22,10 +22,7 @@
<artifactId>tika-parser-web-module</artifactId>
<name>Apache Tika parser web module</name>
<url>http://tika.apache.org/</url>
-
- <properties>
- <mime4j.version>0.7.2</mime4j.version>
- </properties>
+
<dependencies>
<dependency>
http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java b/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
index b42eb89..458ed01 100644
--- a/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
+++ b/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
@@ -51,6 +51,7 @@ import org.apache.tika.config.TikaConfig;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.ParsingEmbeddedDocumentExtractor;
import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Message;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.AutoDetectParser;
@@ -238,6 +239,7 @@ class MailContentHandler implements ContentHandler {
try {
String fieldname = field.getName();
+
ParsedField parsedField = LenientFieldParser.getParser().parse(
field, DecodeMonitor.SILENT);
if (fieldname.equalsIgnoreCase("From")) {
@@ -276,6 +278,9 @@ class MailContentHandler implements ContentHandler {
date = tryOtherDateFormats(field.getBody());
}
metadata.set(TikaCoreProperties.CREATED, date);
+ } else {
+ metadata.add(Metadata.MESSAGE_RAW_HEADER_PREFIX+parsedField.getName(),
+ field.getBody());
}
} catch (RuntimeException me) {
if (strictParsing) {
http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-parser-modules/tika-parser-web-module/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-web-module/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java b/tika-parser-modules/tika-parser-web-module/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
index 3be1edd..4a506be 100644
--- a/tika-parser-modules/tika-parser-web-module/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
+++ b/tika-parser-modules/tika-parser-web-module/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
@@ -355,6 +355,7 @@ public class RFC822ParserTest extends TikaTest {
assertContains("TEST DATA FOR TIKA.", handler.toString());
assertContains("This is text inside an unencrypted zip file", handler.toString());
assertContains("TIKA-1028", handler.toString());
+ assertEquals("<ju...@gmail.com>", metadata.get("Message:Raw-Header:Return-Path"));
}
/**