You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2016/05/17 19:51:57 UTC
[1/3] tika git commit: TIKA-1971 - add another magic for rfc822
Repository: tika
Updated Branches:
refs/heads/master b2821d921 -> de6dbd0a7
TIKA-1971 - add another magic for rfc822
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/e08d0065
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/e08d0065
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/e08d0065
Branch: refs/heads/master
Commit: e08d00654a77267686a112e0665c324ef041c033
Parents: bc0b1f7
Author: tballison <ta...@mitre.org>
Authored: Tue May 17 10:05:35 2016 -0400
Committer: tballison <ta...@mitre.org>
Committed: Tue May 17 10:05:35 2016 -0400
----------------------------------------------------------------------
.../org/apache/tika/mime/tika-mimetypes.xml | 12 +++++++
.../tika/parser/mail/RFC822ParserTest.java | 11 +++++++
.../test-documents/testRFC822_date_utf8 | 8 +++++
.../resources/test-documents/testRFC822_eml | 33 ++++++++++++++++++++
4 files changed, 64 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/e08d0065/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
----------------------------------------------------------------------
diff --git a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
index d82bd95..11b2c3f 100644
--- a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
+++ b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
@@ -679,6 +679,17 @@
<sub-class-of type="text/plain"/>
</mime-type>
+ <mime-type type="application/x-dbf">
+ <_comment>DBF/dbase 3 File</_comment>
+ <magic priority="100">
+ <match value="(?s)^[\x02\x03\x30\x31\x32\x43\x63\x83\x8B\xCB\xF5\xE5\xFB].[\x01-\x0C][\x01-\x1F].{4}(?:.[^\x00]|[\x41-\xFF].)(?:[^\x00\x01].|.[^\x00]).{31}([\x00][^\x00]{0,10})[BCDFGILMNOPQTVWXY@+]" type="regex" offset="0"/>
+ </magic>
+ <glob pattern="*.dbf"/>
+ <glob pattern="*.dbase"/>
+ <glob pattern="*.dbase3"/>
+ <sub-class-of type="text/plain"/>
+ </mime-type>
+
<mime-type type="application/rtx"/>
<mime-type type="application/samlassertion+xml"/>
<mime-type type="application/samlmetadata+xml"/>
@@ -5299,6 +5310,7 @@
<mime-type type="message/rfc822">
<magic priority="50">
+ <match value="Delivered-To:" type="string" offset="0"/>
<match value="Status:" type="string" offset="0"/>
<match value="X-Mozilla-Status:" type="string" offset="0"/>
<match value="X-Mozilla-Status2:" type="string" offset="0"/>
http://git-wip-us.apache.org/repos/asf/tika/blob/e08d0065/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
index a80e544..0e8f613 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
@@ -381,4 +381,15 @@ public class RFC822ParserTest extends TikaTest {
assertEquals(MediaType.TEXT_HTML, tracker.mediaTypes.get(1));
assertEquals(MediaType.image("gif"), tracker.mediaTypes.get(2));
}
+
+ @Test
+ public void testDetection() throws Exception {
+ //test simple text file
+ XMLResult r = getXML("testRFC822_date_utf8");
+ assertEquals("message/rfc822", r.metadata.get(Metadata.CONTENT_TYPE));
+
+ //test without extension
+ r = getXML("testRFC822_eml");
+ assertEquals("message/rfc822", r.metadata.get(Metadata.CONTENT_TYPE));
+ }
}
http://git-wip-us.apache.org/repos/asf/tika/blob/e08d0065/tika-parsers/src/test/resources/test-documents/testRFC822_date_utf8
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/resources/test-documents/testRFC822_date_utf8 b/tika-parsers/src/test/resources/test-documents/testRFC822_date_utf8
new file mode 100644
index 0000000..c3fd0b2
--- /dev/null
+++ b/tika-parsers/src/test/resources/test-documents/testRFC822_date_utf8
@@ -0,0 +1,8 @@
+From: Philipp Steinkr�ger <ph...@uni-koeln.de>
+Subject: Testemail
+Date: 16 May 2016 at 09:30:32 GMT+1
+To: Philipp Steinkr�ger <ph...@googlemail.com>
+
+
+ GMT+1
+
http://git-wip-us.apache.org/repos/asf/tika/blob/e08d0065/tika-parsers/src/test/resources/test-documents/testRFC822_eml
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/resources/test-documents/testRFC822_eml b/tika-parsers/src/test/resources/test-documents/testRFC822_eml
new file mode 100644
index 0000000..5ae32ae
--- /dev/null
+++ b/tika-parsers/src/test/resources/test-documents/testRFC822_eml
@@ -0,0 +1,33 @@
+Delivered-To: philipp.steinkrueger@gmail.com
+Received: by 10.114.185.229 with SMTP id ff5csp1359148ldc;
+ Mon, 16 May 2016 01:30:35 -0700 (PDT)
+X-Received: by 10.194.165.226 with SMTP id zb2mr30717859wjb.172.1463387435671;
+ Mon, 16 May 2016 01:30:35 -0700 (PDT)
+Return-Path: <ph...@uni-koeln.de>
+Received: from smtp-out.rrz.uni-koeln.de (smtp-out.rrz.uni-koeln.de. [2a00:a200:0:12::25])
+ by mx.google.com with ESMTPS id tk4si20973899wjb.25.2016.05.16.01.30.35
+ for <ph...@gmail.com>
+ (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);
+ Mon, 16 May 2016 01:30:35 -0700 (PDT)
+Received-SPF: pass (google.com: domain of philipp.steinkrueger@uni-koeln.de designates 2a00:a200:0:12::25 as permitted sender) client-ip=2a00:a200:0:12::25;
+Authentication-Results: mx.google.com;
+ spf=pass (google.com: domain of philipp.steinkrueger@uni-koeln.de designates 2a00:a200:0:12::25 as permitted sender) smtp.mailfrom=philipp.steinkrueger@uni-koeln.de
+Received: from smtp-auth.rrz.uni-koeln.de (smtp-auth.rrz.uni-koeln.de [IPv6:2a00:a200:0:10::27] (may be forged))
+ by smtp-out.rrz.uni-koeln.de (8.14.4/8.14.4) with ESMTP id u4G8UYXw029242
+ for <ph...@googlemail.com>; Mon, 16 May 2016 10:30:34 +0200
+Received: from [192.168.1.10] (79-66-108-216.dynamic.dsl.as9105.com [79.66.108.216])
+ (authenticated as user altj4 using DIGEST-MD5 bits=0)
+ by smtp-auth.uni-koeln.de (8.13.8/8.13.8) with ESMTP id u4G8UXlP028450
+ (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=NO)
+ for <ph...@googlemail.com>; Mon, 16 May 2016 10:30:34 +0200
+From: =?utf-8?Q?Philipp_Steinkr=C3=BCger?= <ph...@uni-koeln.de>
+Content-Type: text/plain
+Content-Transfer-Encoding: 7bit
+Subject: Testemail
+Message-Id: <0B...@uni-koeln.de>
+Date: Mon, 16 May 2016 09:30:32 +0100
+To: =?utf-8?Q?Philipp_Steinkr=C3=BCger?= <ph...@googlemail.com>
+Mime-Version: 1.0 (Mac OS X Mail 9.3 \(3124\))
+X-Mailer: Apple Mail (2.3124)
+X-Scanned-By: MIMEDefang 2.75
+
[2/3] tika git commit: TIKA-1970 - Mac Mail date of interesting
format not parsed by james mime4j
Posted by ta...@apache.org.
TIKA-1970 - Mac Mail date of interesting format not parsed by james mime4j
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/09cc658a
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/09cc658a
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/09cc658a
Branch: refs/heads/master
Commit: 09cc658a30ae194323f1e444be73d396544efc6b
Parents: e08d006
Author: tballison <ta...@mitre.org>
Authored: Tue May 17 10:54:02 2016 -0400
Committer: tballison <ta...@mitre.org>
Committed: Tue May 17 10:54:02 2016 -0400
----------------------------------------------------------------------
.../tika/parser/mail/MailContentHandler.java | 62 +++++++++++++++++++-
.../tika/parser/mail/RFC822ParserTest.java | 15 +++++
2 files changed, 76 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/09cc658a/tika-parsers/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java b/tika-parsers/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
index 5369c1d..5a6984c 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
@@ -18,6 +18,15 @@ package org.apache.tika.parser.mail;
import java.io.IOException;
import java.io.InputStream;
+import java.text.DateFormat;
+import java.text.DateFormatSymbols;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Locale;
+import java.util.TimeZone;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.james.mime4j.MimeException;
import org.apache.james.mime4j.codec.DecodeMonitor;
@@ -35,9 +44,12 @@ import org.apache.james.mime4j.field.LenientFieldParser;
import org.apache.james.mime4j.parser.ContentHandler;
import org.apache.james.mime4j.stream.BodyDescriptor;
import org.apache.james.mime4j.stream.Field;
+import org.apache.james.mime4j.util.ByteSequence;
import org.apache.tika.config.TikaConfig;
+import org.apache.tika.detect.AutoDetectReader;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.ParsingEmbeddedDocumentExtractor;
+import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.AutoDetectParser;
@@ -46,6 +58,9 @@ import org.apache.tika.parser.Parser;
import org.apache.tika.sax.XHTMLContentHandler;
import org.xml.sax.SAXException;
+import static org.apache.tika.utils.DateUtils.MIDDAY;
+import static org.apache.tika.utils.DateUtils.UTC;
+
/**
* Bridge between mime4j's content handler and the generic Sax content handler
* used by Tika. See
@@ -53,6 +68,27 @@ import org.xml.sax.SAXException;
*/
class MailContentHandler implements ContentHandler {
+ //TIKA-1970 Mac Mail's format
+ private static final Pattern GENERAL_TIME_ZONE_NO_MINUTES_PATTERN =
+ Pattern.compile("(?:UTC|GMT)([+-])(\\d?\\d)\\Z");
+
+ private static final DateFormat[] ALTERNATE_DATE_FORMATS = new DateFormat[] {
+ //16 May 2016 at 09:30:32 GMT+1
+ createDateFormat("dd MMM yyyy 'at' HH:mm:ss z", UTC), // UTC/Zulu
+ };
+
+ private static DateFormat createDateFormat(String format, TimeZone timezone) {
+ SimpleDateFormat sdf =
+ new SimpleDateFormat(format, new DateFormatSymbols(Locale.US));
+ if (timezone != null) {
+ sdf.setTimeZone(timezone);
+ }
+ return sdf;
+ }
+
+
+
+
private boolean strictParsing = false;
private XHTMLContentHandler handler;
@@ -197,15 +233,39 @@ class MailContentHandler implements ContentHandler {
processAddressList(parsedField, "Bcc:", Metadata.MESSAGE_BCC);
} else if (fieldname.equalsIgnoreCase("Date")) {
DateTimeField dateField = (DateTimeField) parsedField;
- metadata.set(TikaCoreProperties.CREATED, dateField.getDate());
+ Date date = dateField.getDate();
+ if (date == null) {
+ date = tryOtherDateFormats(field.getBody());
+ }
+ metadata.set(TikaCoreProperties.CREATED, date);
}
} catch (RuntimeException me) {
+ me.printStackTrace();
if (strictParsing) {
throw me;
}
}
}
+ private static synchronized Date tryOtherDateFormats(String text) {
+ if (text == null) {
+ return null;
+ }
+ text = text.replaceAll("\\s+", " ").trim();
+ Matcher matcher = GENERAL_TIME_ZONE_NO_MINUTES_PATTERN.matcher(text);
+ if (matcher.find()) {
+ text = matcher.replaceFirst("GMT$1$2:00");
+ }
+
+ for (DateFormat format : ALTERNATE_DATE_FORMATS) {
+ try {
+ return format.parse(text);
+ } catch (ParseException e) {
+ }
+ }
+ return null;
+ }
+
private void processAddressList(ParsedField field, String addressListType,
String metadataField) throws MimeException {
AddressListField toField = (AddressListField) field;
http://git-wip-us.apache.org/repos/asf/tika/blob/09cc658a/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
index 0e8f613..6a69ea5 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
@@ -32,6 +32,9 @@ import static org.mockito.Mockito.verify;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.TimeZone;
import org.apache.james.mime4j.stream.MimeConfig;
import org.apache.tika.TikaTest;
@@ -392,4 +395,16 @@ public class RFC822ParserTest extends TikaTest {
r = getXML("testRFC822_eml");
assertEquals("message/rfc822", r.metadata.get(Metadata.CONTENT_TYPE));
}
+
+ @Test
+ public void testDates() throws Exception {
+
+ //tests non-standard dates that mime4j can't parse
+ XMLResult r = getXML("testRFC822_date_utf8");
+ assertEquals("2016-05-16T08:30:32Z", r.metadata.get(TikaCoreProperties.CREATED));
+
+ r = getXML("testRFC822_eml");
+ assertEquals("2016-05-16T08:30:32Z", r.metadata.get(TikaCoreProperties.CREATED));
+ }
+
}
RE: [1/3] tika git commit: TIKA-1971 - add another magic for rfc822
Posted by "Allison, Timothy B." <ta...@mitre.org>.
Doh! Sorry. Fixed. Thank you, Nick.
-----Original Message-----
From: Nick Burch [mailto:apache@gagravarr.org]
Sent: Tuesday, May 17, 2016 5:03 PM
To: dev@tika.apache.org
Subject: Re: [1/3] tika git commit: TIKA-1971 - add another magic for rfc822
On Tue, 17 May 2016, tallison@apache.org wrote:
> TIKA-1971 - add another magic for rfc822
>
>
> http://git-wip-us.apache.org/repos/asf/tika/blob/e08d0065/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
> ----------------------------------------------------------------------
> diff --git a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
> index d82bd95..11b2c3f 100644
> --- a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
> +++ b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
> @@ -679,6 +679,17 @@
> <sub-class-of type="text/plain"/>
> </mime-type>
>
> + <mime-type type="application/x-dbf">
> + <_comment>DBF/dbase 3 File</_comment>
Did you mean to include this in the rfc822 fix?
Thanks
Nick
Re: [1/3] tika git commit: TIKA-1971 - add another magic for
rfc822
Posted by Nick Burch <ap...@gagravarr.org>.
On Tue, 17 May 2016, tallison@apache.org wrote:
> TIKA-1971 - add another magic for rfc822
>
>
> http://git-wip-us.apache.org/repos/asf/tika/blob/e08d0065/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
> ----------------------------------------------------------------------
> diff --git a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
> index d82bd95..11b2c3f 100644
> --- a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
> +++ b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
> @@ -679,6 +679,17 @@
> <sub-class-of type="text/plain"/>
> </mime-type>
>
> + <mime-type type="application/x-dbf">
> + <_comment>DBF/dbase 3 File</_comment>
Did you mean to include this in the rfc822 fix?
Thanks
Nick
[3/3] tika git commit: Merge remote-tracking branch 'origin/master'
Posted by ta...@apache.org.
Merge remote-tracking branch 'origin/master'
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/de6dbd0a
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/de6dbd0a
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/de6dbd0a
Branch: refs/heads/master
Commit: de6dbd0a7fbb2e24c357020e51f28d21c2ad2ebf
Parents: 09cc658 b2821d9
Author: tballison <ta...@mitre.org>
Authored: Tue May 17 15:51:17 2016 -0400
Committer: tballison <ta...@mitre.org>
Committed: Tue May 17 15:51:17 2016 -0400
----------------------------------------------------------------------
.../java/org/apache/tika/detect/NameDetector.java | 15 ++++++++++-----
.../org/apache/tika/detect/NameDetectorTest.java | 10 ++++++++++
2 files changed, 20 insertions(+), 5 deletions(-)
----------------------------------------------------------------------