You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2017/03/02 14:38:34 UTC
[tika] branch 2.x updated: TIKA-1865 clean up, deduplicate MailUtil,
bug fix
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch 2.x
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/2.x by this push:
new 70895fc TIKA-1865 clean up, deduplicate MailUtil, bug fix
70895fc is described below
commit 70895fcd9ae84d92d0c5c1a15a426eb68ff4b52a
Author: tballison <ta...@mitre.org>
AuthorDate: Thu Mar 2 09:38:26 2017 -0500
TIKA-1865 clean up, deduplicate MailUtil, bug fix
---
.../tika-parser-office-bundle/pom.xml | 1 +
tika-parser-bundles/tika-parser-web-bundle/pom.xml | 1 +
tika-parser-modules/pom.xml | 1 +
.../pom.xml | 47 +--------
.../apache/tika/parser/mail/util}/MailUtil.java | 2 +-
.../tika/parser/mail/util}/MailUtilTest.java | 2 +-
.../tika-parser-office-module/pom.xml | 5 +
.../org/apache/tika/parser/mbox/MboxParser.java | 2 +-
.../org/apache/tika/parser/microsoft/MailUtil.java | 115 ---------------------
.../tika/parser/microsoft/OutlookExtractor.java | 1 +
tika-parser-modules/tika-parser-web-module/pom.xml | 5 +
.../tika/parser/mail/MailContentHandler.java | 1 +
.../org/apache/tika/parser/mail/RFC822Parser.java | 2 +-
13 files changed, 21 insertions(+), 164 deletions(-)
diff --git a/tika-parser-bundles/tika-parser-office-bundle/pom.xml b/tika-parser-bundles/tika-parser-office-bundle/pom.xml
index 7acddad..727d9c9 100644
--- a/tika-parser-bundles/tika-parser-office-bundle/pom.xml
+++ b/tika-parser-bundles/tika-parser-office-bundle/pom.xml
@@ -62,6 +62,7 @@
<Bundle-Activator>org.apache.tika.module.office.internal.Activator</Bundle-Activator>
<Embed-Dependency>
tika-parser-office-module;inline=true,
+ tika-parser-mail-commons;inline=true,
commons-lang;inline=true,
commons-io;inline=true,
commons-codec;inline=true,
diff --git a/tika-parser-bundles/tika-parser-web-bundle/pom.xml b/tika-parser-bundles/tika-parser-web-bundle/pom.xml
index a23267d..8d59bb3 100644
--- a/tika-parser-bundles/tika-parser-web-bundle/pom.xml
+++ b/tika-parser-bundles/tika-parser-web-bundle/pom.xml
@@ -47,6 +47,7 @@
<Bundle-Activator>org.apache.tika.module.web.internal.Activator</Bundle-Activator>
<Embed-Dependency>
tika-parser-web-module;inline=true,
+ tika-parser-mail-commons;inline=true,
tagsoup;inline=true,
boilerpipe;inline=true,
rome;inline=true,
diff --git a/tika-parser-modules/pom.xml b/tika-parser-modules/pom.xml
index af70259..e6c1952 100644
--- a/tika-parser-modules/pom.xml
+++ b/tika-parser-modules/pom.xml
@@ -54,6 +54,7 @@
<module>tika-parser-database-module</module>
<module>tika-parser-ebook-module</module>
<module>tika-parser-journal-module</module>
+ <module>tika-parser-mail-commons</module>
<module>tika-parser-multimedia-module</module>
<module>tika-parser-office-module</module>
<module>tika-parser-package-module</module>
diff --git a/tika-parser-modules/tika-parser-web-module/pom.xml b/tika-parser-modules/tika-parser-mail-commons/pom.xml
similarity index 53%
copy from tika-parser-modules/tika-parser-web-module/pom.xml
copy to tika-parser-modules/tika-parser-mail-commons/pom.xml
index 6f11184..4899656 100644
--- a/tika-parser-modules/tika-parser-web-module/pom.xml
+++ b/tika-parser-modules/tika-parser-mail-commons/pom.xml
@@ -19,10 +19,9 @@
<version>2.0-SNAPSHOT</version>
</parent>
- <artifactId>tika-parser-web-module</artifactId>
- <name>Apache Tika parser web module</name>
+ <artifactId>tika-parser-mail-commons</artifactId>
+ <name>Apache Tika parser mail commons</name>
<url>http://tika.apache.org/</url>
-
<dependencies>
<dependency>
@@ -30,48 +29,6 @@
<artifactId>tika-core</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.ccil.cowan.tagsoup</groupId>
- <artifactId>tagsoup</artifactId>
- <version>1.2.1</version>
- </dependency>
- <dependency>
- <groupId>de.l3s.boilerpipe</groupId>
- <artifactId>boilerpipe</artifactId>
- <version>1.1.0</version>
- </dependency>
- <dependency>
- <groupId>com.rometools</groupId>
- <artifactId>rome</artifactId>
- <version>1.5.1</version>
- </dependency>
- <dependency>
- <groupId>org.apache.james</groupId>
- <artifactId>apache-mime4j-core</artifactId>
- <version>${mime4j.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.james</groupId>
- <artifactId>apache-mime4j-dom</artifactId>
- <version>${mime4j.version}</version>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>${commons.io.version}</version>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-parser-text-module</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-parser-package-module</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
</dependencies>
<build>
diff --git a/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/MailUtil.java b/tika-parser-modules/tika-parser-mail-commons/src/main/java/org/apache/tika/parser/mail/util/MailUtil.java
similarity index 98%
rename from tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/MailUtil.java
rename to tika-parser-modules/tika-parser-mail-commons/src/main/java/org/apache/tika/parser/mail/util/MailUtil.java
index 1cf2686..6c03f8a 100644
--- a/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/MailUtil.java
+++ b/tika-parser-modules/tika-parser-mail-commons/src/main/java/org/apache/tika/parser/mail/util/MailUtil.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.tika.parser.mail;
+package org.apache.tika.parser.mail.util;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/mail/MailUtilTest.java b/tika-parser-modules/tika-parser-mail-commons/src/test/java/org/apache/tika/parser/mail/util/MailUtilTest.java
similarity index 98%
rename from tika-parsers/src/test/java/org/apache/tika/parser/mail/MailUtilTest.java
rename to tika-parser-modules/tika-parser-mail-commons/src/test/java/org/apache/tika/parser/mail/util/MailUtilTest.java
index 3143ce6..debb4ab 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/mail/MailUtilTest.java
+++ b/tika-parser-modules/tika-parser-mail-commons/src/test/java/org/apache/tika/parser/mail/util/MailUtilTest.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.tika.parser.mail;
+package org.apache.tika.parser.mail.util;
import static org.junit.Assert.assertEquals;
diff --git a/tika-parser-modules/tika-parser-office-module/pom.xml b/tika-parser-modules/tika-parser-office-module/pom.xml
index d2cba28..96500ac 100644
--- a/tika-parser-modules/tika-parser-office-module/pom.xml
+++ b/tika-parser-modules/tika-parser-office-module/pom.xml
@@ -30,6 +30,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-parser-mail-commons</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>${commons.io.version}</version>
diff --git a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/mbox/MboxParser.java b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/mbox/MboxParser.java
index 76a0de1..696ed60 100644
--- a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/mbox/MboxParser.java
+++ b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/mbox/MboxParser.java
@@ -44,7 +44,7 @@ import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.AbstractParser;
import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.microsoft.MailUtil;
+import org.apache.tika.parser.mail.util.MailUtil;
import org.apache.tika.sax.XHTMLContentHandler;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
diff --git a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/MailUtil.java b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/MailUtil.java
deleted file mode 100644
index a61afe2..0000000
--- a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/MailUtil.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.parser.microsoft;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.Property;
-
-//TODO: figure out if there's a way to not have to duplicate this with the web module!
-public class MailUtil {
-
- private final static Pattern EMAIL = Pattern.compile("(?i)([^<@\\s]+@[^@> ]+)");
-
- /**
- * This tries to split a "from" or "to" value into a person field and an email field.
- * This does not handle RFC encoded strings (e.g. "=?iso-8859-1?Q?H=E9roux_Louise?"),
- * you must decode them first.
- *
- * @param string
- * @param personProperty
- * @param emailProperty
- * @param metadata
- */
- public static void setPersonAndEmail(String string,
- Property personProperty,
- Property emailProperty, Metadata metadata) {
-
- StringBuffer sb = new StringBuffer();
- String email = extractEmail(string, sb);
- String person = clean(sb.toString());
- if (person != null && person.length() > 0) {
- metadata.set(personProperty, person);
- }
- if (email != null && email.length() > 0) {
- metadata.set(emailProperty, email);
- }
-
- }
-
- /**
- * This tries to split a "from" or "to" value into a person field and an email field.
- * This does not handle RFC encoded strings (e.g. "=?iso-8859-1?Q?H=E9roux_Louise?"),
- * you must decode them first.
- *
- * @param string
- * @param personProperty
- * @param emailProperty
- * @param metadata
- */
- public static void addPersonAndEmail(String string,
- Property personProperty,
- Property emailProperty, Metadata metadata) {
-
- StringBuffer sb = new StringBuffer();
- String email = extractEmail(string, sb);
- String person = clean(sb.toString());
-
- if (person != null && person.length() > 0) {
- metadata.add(personProperty, person);
- }
- if (email != null && email.length() > 0) {
- metadata.add(emailProperty, email);
- }
-
- }
-
- private static String clean(String s) {
- s = s.replaceAll("[<>\"]", " ");
- s = s.trim();
- return s;
- }
-
- private static String extractEmail(String string, StringBuffer sb) {
- Matcher emailMatcher = EMAIL.matcher(string);
- String email = "";
- while (emailMatcher.find()) {
- emailMatcher.appendReplacement(sb, "");
- email = emailMatcher.group(1);
- }
- return email;
- }
-
- /**
- * If the chunk looks like it contains an email
- * @param chunk
- * @return
- */
- public static boolean containsEmail(String chunk) {
- if (chunk == null) {
- return false;
- }
-
- if (EMAIL.matcher(chunk).find()) {
- return true;
- }
- return false;
- }
-}
diff --git a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
index 553d8d1..55e4ad6 100644
--- a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
+++ b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
@@ -64,6 +64,7 @@ import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.ParserProxy;
+import org.apache.tika.parser.mail.util.MailUtil;
import org.apache.tika.parser.mbox.MboxParser;
import org.apache.tika.parser.rtf.RTFParser;
import org.apache.tika.parser.txt.CharsetDetector;
diff --git a/tika-parser-modules/tika-parser-web-module/pom.xml b/tika-parser-modules/tika-parser-web-module/pom.xml
index 6f11184..2155acc 100644
--- a/tika-parser-modules/tika-parser-web-module/pom.xml
+++ b/tika-parser-modules/tika-parser-web-module/pom.xml
@@ -31,6 +31,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-parser-mail-commons</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.ccil.cowan.tagsoup</groupId>
<artifactId>tagsoup</artifactId>
<version>1.2.1</version>
diff --git a/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java b/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
index 71be646..597fa16 100644
--- a/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
+++ b/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
@@ -54,6 +54,7 @@ import org.apache.tika.metadata.Message;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.mail.util.MailUtil;
import org.apache.tika.sax.XHTMLContentHandler;
import org.xml.sax.SAXException;
diff --git a/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/RFC822Parser.java b/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/RFC822Parser.java
index 6299d3f..2a6ba06 100644
--- a/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/RFC822Parser.java
+++ b/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/RFC822Parser.java
@@ -38,7 +38,7 @@ import org.xml.sax.SAXException;
* Uses apache-mime4j to parse emails. Each part is treated with the
* corresponding parser and displayed within elements.
* <p/>
- * A {@link MimeEntityConfig} object can be passed in the parsing context
+ * A {@link ParseContext} object can be passed in the parsing context
* to better control the parsing process.
*
* @author jnioche@digitalpebble.com
--
To stop receiving notification emails like this one, please contact
['"commits@tika.apache.org" <co...@tika.apache.org>'].