You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2020/08/12 09:58:22 UTC
[tika] branch main updated (7c68eb2 -> 4e05727)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git.
from 7c68eb2 Merge pull request #337 from JoaoGFarias/patch-1
new 811330f TIKA-3160 -- run surefire tests headless
new 4e05727 TIKA-3153 -- add MinShouldMatchClause to allow more flexibility in defining rfc822 detection
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/tika/mime/MimeTypesReader.java | 57 ++++++++++++---
.../apache/tika/mime/MimeTypesReaderMetKeys.java | 2 +
.../{OrClause.java => MinShouldMatchClause.java} | 35 ++++++++--
.../org/apache/tika/mime/tika-mimetypes.xml | 81 +++++++++++++---------
.../org/apache/tika/mime/MimeTypesReaderTest.java | 73 ++++++++++++++++++-
...xml => custom-mimetypes-badMinShouldMatch1.xml} | 21 ++++--
...xml => custom-mimetypes-badMinShouldMatch2.xml} | 21 ++++--
...xml => custom-mimetypes-badMinShouldMatch3.xml} | 21 ++++--
...xml => custom-mimetypes-badMinShouldMatch4.xml} | 16 +++--
.../custom-mimetypes-minShouldMatch.xml} | 30 +++++---
tika-parent/pom.xml | 2 +-
.../apache/tika/parser/mail/RFC822ParserTest.java | 2 +-
12 files changed, 288 insertions(+), 73 deletions(-)
copy tika-core/src/main/java/org/apache/tika/mime/{OrClause.java => MinShouldMatchClause.java} (55%)
copy tika-core/src/test/resources/org/apache/tika/mime/{custom-mimetypes2.xml => custom-mimetypes-badMinShouldMatch1.xml} (61%)
copy tika-core/src/test/resources/org/apache/tika/mime/{custom-mimetypes2.xml => custom-mimetypes-badMinShouldMatch2.xml} (61%)
copy tika-core/src/test/resources/org/apache/tika/mime/{custom-mimetypes2.xml => custom-mimetypes-badMinShouldMatch3.xml} (60%)
copy tika-core/src/test/resources/org/apache/tika/mime/{custom-mimetypes2.xml => custom-mimetypes-badMinShouldMatch4.xml} (76%)
copy tika-core/src/test/resources/org/apache/tika/{config/TIKA-1986-bad-values.xml => mime/custom-mimetypes-minShouldMatch.xml} (60%)
[tika] 02/02: TIKA-3153 -- add MinShouldMatchClause to allow more
flexibility in defining rfc822 detection
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 4e057270f075a36e535f55f1542972d6fa566dbf
Author: tballison <ta...@apache.org>
AuthorDate: Wed Aug 12 05:58:08 2020 -0400
TIKA-3153 -- add MinShouldMatchClause to allow more flexibility in defining rfc822 detection
---
.../java/org/apache/tika/mime/MimeTypesReader.java | 57 ++++++++++++---
.../apache/tika/mime/MimeTypesReaderMetKeys.java | 2 +
.../org/apache/tika/mime/MinShouldMatchClause.java | 77 ++++++++++++++++++++
.../org/apache/tika/mime/tika-mimetypes.xml | 81 +++++++++++++---------
.../org/apache/tika/mime/MimeTypesReaderTest.java | 73 ++++++++++++++++++-
.../mime/custom-mimetypes-badMinShouldMatch1.xml | 37 ++++++++++
.../mime/custom-mimetypes-badMinShouldMatch2.xml | 37 ++++++++++
.../mime/custom-mimetypes-badMinShouldMatch3.xml | 37 ++++++++++
.../mime/custom-mimetypes-badMinShouldMatch4.xml | 32 +++++++++
.../tika/mime/custom-mimetypes-minShouldMatch.xml | 38 ++++++++++
.../apache/tika/parser/mail/RFC822ParserTest.java | 2 +-
11 files changed, 430 insertions(+), 43 deletions(-)
diff --git a/tika-core/src/main/java/org/apache/tika/mime/MimeTypesReader.java b/tika-core/src/main/java/org/apache/tika/mime/MimeTypesReader.java
index adc50de..3dff861 100644
--- a/tika-core/src/main/java/org/apache/tika/mime/MimeTypesReader.java
+++ b/tika-core/src/main/java/org/apache/tika/mime/MimeTypesReader.java
@@ -204,15 +204,22 @@ public class MimeTypesReader extends DefaultHandler implements MimeTypesReaderMe
String name = attributes.getValue(LOCAL_NAME_ATTR);
type.addRootXML(namespace, name);
} else if (MATCH_TAG.equals(qName)) {
- String kind = attributes.getValue(MATCH_TYPE_ATTR);
- String offset = attributes.getValue(MATCH_OFFSET_ATTR);
- String value = attributes.getValue(MATCH_VALUE_ATTR);
- String mask = attributes.getValue(MATCH_MASK_ATTR);
- if (kind == null) {
- kind = "string";
+ if (attributes.getValue(MATCH_MINSHOULDMATCH_ATTR) != null) {
+ current = new ClauseRecord(
+ new MinShouldMatchVal(
+ Integer.parseInt(
+ attributes.getValue(MATCH_MINSHOULDMATCH_ATTR))));
+ } else {
+ String kind = attributes.getValue(MATCH_TYPE_ATTR);
+ String offset = attributes.getValue(MATCH_OFFSET_ATTR);
+ String value = attributes.getValue(MATCH_VALUE_ATTR);
+ String mask = attributes.getValue(MATCH_MASK_ATTR);
+ if (kind == null) {
+ kind = "string";
+ }
+ current = new ClauseRecord(
+ new MagicMatch(type.getType(), kind, offset, value, mask));
}
- current = new ClauseRecord(
- new MagicMatch(type.getType(), kind, offset, value, mask));
} else if (MAGIC_TAG.equals(qName)) {
String value = attributes.getValue(MAGIC_PRIORITY_ATTR);
if (value != null && value.length() > 0) {
@@ -288,7 +295,9 @@ public class MimeTypesReader extends DefaultHandler implements MimeTypesReaderMe
}
public void stop() {
- if (subclauses != null) {
+ if (clause instanceof MinShouldMatchVal) {
+ clause = new MinShouldMatchClause(((MinShouldMatchVal)clause).getVal(), subclauses);
+ } else if (subclauses != null) {
Clause subclause;
if (subclauses.size() == 1) {
subclause = subclauses.get(0);
@@ -297,6 +306,7 @@ public class MimeTypesReader extends DefaultHandler implements MimeTypesReaderMe
}
clause = new AndClause(clause, subclause);
}
+
if (parent.subclauses == null) {
parent.subclauses = Collections.singletonList(clause);
} else {
@@ -314,6 +324,35 @@ public class MimeTypesReader extends DefaultHandler implements MimeTypesReaderMe
}
}
+
+ /**
+ * Shim class used during building of actual classes.
+ * This temporarily holds the value of the minShouldMatchClause
+ * so that the actual MinShouldMatchClause can have a cleaner/immutable
+ * initialization.
+ */
+ private static class MinShouldMatchVal implements Clause {
+
+ private final int val;
+
+ MinShouldMatchVal(int val) {
+ this.val = val;
+ }
+
+ int getVal() {
+ return val;
+ }
+ @Override
+ public boolean eval(byte[] data) {
+ throw new IllegalStateException("This should never be used " +
+ "on this placeholder class");
+ }
+
+ @Override
+ public int size() {
+ return 0;
+ }
+ }
/**
* Acquire a SAXParser from the pool; create one if it
* doesn't exist. Make sure to {@link #releaseParser(SAXParser)} in
diff --git a/tika-core/src/main/java/org/apache/tika/mime/MimeTypesReaderMetKeys.java b/tika-core/src/main/java/org/apache/tika/mime/MimeTypesReaderMetKeys.java
index c77cc5c..fdc8b5f 100644
--- a/tika-core/src/main/java/org/apache/tika/mime/MimeTypesReaderMetKeys.java
+++ b/tika-core/src/main/java/org/apache/tika/mime/MimeTypesReaderMetKeys.java
@@ -63,6 +63,8 @@ public interface MimeTypesReaderMetKeys {
String MATCH_MASK_ATTR = "mask";
+ String MATCH_MINSHOULDMATCH_ATTR = "minShouldMatch";
+
String NS_URI_ATTR = "namespaceURI";
String LOCAL_NAME_ATTR = "localName";
diff --git a/tika-core/src/main/java/org/apache/tika/mime/MinShouldMatchClause.java b/tika-core/src/main/java/org/apache/tika/mime/MinShouldMatchClause.java
new file mode 100644
index 0000000..c723385
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/mime/MinShouldMatchClause.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.mime;
+
+import java.util.List;
+
+class MinShouldMatchClause implements Clause {
+
+ private final int min;
+ private final List<Clause> clauses;
+
+ /**
+ * Minimum number of clauses that need to match.
+ *
+ * Throws IllegalArgumentException if min <= 0,
+ * if clauses is null or has size == 0, or if min > clauses.size()
+ *
+ * @param min
+ * @param clauses
+ */
+ MinShouldMatchClause(int min, List<Clause> clauses) {
+
+ if (clauses == null || clauses.size() == 0) {
+ throw new IllegalArgumentException("clauses must be not null with size > 0");
+ }
+
+ if (min > clauses.size()) {
+ throw new IllegalArgumentException("min (" +
+ min+ ") cannot be > clauses.size ("+
+ clauses.size() +")");
+ } else if (min <= 0) {
+ throw new IllegalArgumentException("min cannot be <= 0: "+min);
+ }
+
+ this.min = min;
+ this.clauses = clauses;
+ }
+
+ public boolean eval(byte[] data) {
+ int matches = 0;
+ for (Clause clause : clauses) {
+ if (clause.eval(data)) {
+ if (++matches >= min) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ public int size() {
+ int size = 0;
+ for (Clause clause : clauses) {
+ size = Math.max(size, clause.size());
+ }
+ return size;
+ }
+
+ public String toString() {
+ return "minShouldMatch (min: "+min+") " + clauses;
+ }
+
+}
diff --git a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
index 7e9b455..61d07a2 100644
--- a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
+++ b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
@@ -6135,9 +6135,11 @@
<mime-type type="message/imdn+xml"/>
<mime-type type="message/news">
- <magic priority="50">
+ <magic priority="60">
<match value="Path:" type="string" offset="0" />
<match value="Xref:" type="string" offset="0" />
+ </magic>
+ <magic priority="50">
<match value="Article" type="string" offset="0" />
</magic>
<sub-class-of type="text/x-tika-text-based-message"/>
@@ -6147,50 +6149,65 @@
<mime-type type="message/rfc822">
<magic priority="50">
- <match value="Delivered-To:" type="string" offset="0"/>
- <match value="Status:" type="string" offset="0"/>
+ <!-- these should be 100% hits...if you see this at offset=0 -->
<match value="Relay-Version:" type="stringignorecase" offset="0"/>
<match value="#!\ rnews" type="string" offset="0"/>
<match value="N#!\ rnews" type="string" offset="0"/>
<match value="Forward\ to" type="string" offset="0"/>
<match value="Pipe\ to" type="string" offset="0"/>
<match value="Return-Path:" type="stringignorecase" offset="0"/>
- <match value="From:" type="stringignorecase" offset="0"/>
- <match value="Received:" type="stringignorecase" offset="0"/>
<match value="Message-ID:" type="stringignorecase" offset="0"/>
- <match value="\nReturn-Path:" type="stringignorecase" offset="0:1000"/>
- <match value="\nX-Originating-IP:" type="stringignorecase" offset="0:1000"/>
- <match value="\nReceived:" type="stringignorecase" offset="0:1000"/>
- <match value="Date:" type="string" offset="0"/>
- <match value="User-Agent:" type="string" offset="0"/>
- <match value="MIME-Version:" type="stringignorecase" offset="0"/>
<match value="X-Mailer:" type="string" offset="0"/>
<match value="X-Notes-Item:" type="string" offset="0">
<match value="Message-ID:" type="string" offset="0:8192"/>
</match>
- <match value="X-" type="stringignorecase" offset="0">
- <match value="\nMessage-ID:" type="string" offset="0:8192"/>
- <match value="\nFrom:" type="stringignorecase" offset="0:8192"/>
- <match value="\nTo:" type="stringignorecase" offset="0:8192"/>
- <match value="\nSubject:" type="string" offset="0:8192"/>
- <match value="\nReceived:" type="string" offset="0:8192"/>
- <match value="\nMIME-Version:" type="stringignorecase" offset="0:8192"/>
- </match>
- <match value="DKIM-" type="string" offset="0">
- <match value="\nMessage-ID:" type="string" offset="0:8192"/>
- <match value="\nFrom:" type="stringignorecase" offset="0:8192"/>
- <match value="\nTo:" type="stringignorecase" offset="0:8192"/>
- <match value="\nSubject:" type="string" offset="0:8192"/>
- <match value="\nReceived:" type="string" offset="0:8192"/>
- <match value="\nMIME-Version:" type="stringignorecase" offset="0:8192"/>
- </match>
- <match value="ARC-" type="string" offset="0">
- <match value="\nMessage-ID:" type="string" offset="0:8192"/>
+ <!-- be a bit more flexible, but require two of these -->
+ <match minShouldMatch="2">
+ <match value="Date:" type="stringignorecase" offset="0"/>
+ <match value="Delivered-To:" type="string" offset="0"/>
+ <match value="From:" type="stringignorecase" offset="0"/>
+ <match value="Message-ID:" type="stringignorecase" offset="0"/>
+ <match value="MIME-Version:" type="stringignorecase" offset="0"/>
+ <match value="Received:" type="stringignorecase" offset="0"/>
+ <match value="Relay-Version:" type="stringignorecase" offset="0"/>
+ <match value="Return-Path:" type="stringignorecase" offset="0"/>
+ <match value="Sent:" type="string" offset="0"/>
+ <match value="Status:" type="string" offset="0"/>
+ <match value="User-Agent:" type="string" offset="0"/>
+ <match value="X-Mailer:" type="string" offset="0"/>
+ <match value="X-Originating-IP:" type="stringignorecase" offset="0"/>
+
+ <match value="\nDate:" type="stringignorecase" offset="0:1024"/>
+ <match value="\nDelivered-To:" type="string" offset="0:1024"/>
+ <match value="\nFrom:" type="stringignorecase" offset="0:1024"/>
+ <match value="\nMIME-Version:" type="stringignorecase" offset="0:1024"/>
+ <match value="\nReceived:" type="stringignorecase" offset="0:1024"/>
+ <match value="\nRelay-Version:" type="stringignorecase" offset="0:1024"/>
+ <match value="\nReturn-Path:" type="stringignorecase" offset="0:1024"/>
+ <match value="\nSent:" type="string" offset="0:1024"/>
+ <match value="\nStatus:" type="string" offset="0:1024"/>
+ <match value="\nSubject:" type="string" offset="0:1024"/>
+ <match value="\nTo:" type="string" offset="0:1024"/>
+ <match value="\nUser-Agent:" type="string" offset="0:1024"/>
+ <match value="\nX-Mailer:" type="string" offset="0:1024"/>
+ <match value="\nX-Originating-IP:" type="stringignorecase" offset="0:1024"/>
+ </match>
+ <!-- match X- DKIM- ARC- at start of file and then require at least one
+ of the usual: from, received, date...but look farther into the file
+ because of the X|DKIM|ARC headers-->
+ <match value="(X|DKIM|ARC)-" type="regex" offset="0">
+ <match value="\nDate:" type="stringignorecase" offset="0:8192"/>
+ <match value="\nDelivered-To:" type="string" offset="0:8192"/>
<match value="\nFrom:" type="stringignorecase" offset="0:8192"/>
- <match value="\nTo:" type="stringignorecase" offset="0:8192"/>
- <match value="\nSubject:" type="string" offset="0:8192"/>
- <match value="\nReceived:" type="string" offset="0:8192"/>
+ <match value="\nMessage-ID:" type="stringignorecase" offset="0:8192"/>
<match value="\nMIME-Version:" type="stringignorecase" offset="0:8192"/>
+ <match value="\nReceived:" type="stringignorecase" offset="0:8192"/>
+ <match value="\nRelay-Version:" type="stringignorecase" offset="0:8192"/>
+ <match value="\nReturn-Path:" type="stringignorecase" offset="0:8192"/>
+ <match value="\nStatus:" type="string" offset="0:8192"/>
+ <match value="\nUser-Agent:" type="string" offset="0:8192"/>
+ <match value="\nX-Mailer:" type="string" offset="0:8192"/>
+ <match value="\nX-Originating-IP:" type="stringignorecase" offset="0:8192"/>
</match>
</magic>
<magic priority="40">
diff --git a/tika-core/src/test/java/org/apache/tika/mime/MimeTypesReaderTest.java b/tika-core/src/test/java/org/apache/tika/mime/MimeTypesReaderTest.java
index a511538..c688688 100644
--- a/tika-core/src/test/java/org/apache/tika/mime/MimeTypesReaderTest.java
+++ b/tika-core/src/test/java/org/apache/tika/mime/MimeTypesReaderTest.java
@@ -23,7 +23,9 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
+import java.io.IOException;
import java.lang.reflect.Field;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.ConcurrentModificationException;
import java.util.List;
@@ -58,7 +60,8 @@ public class MimeTypesReaderTest {
@SuppressWarnings("unchecked")
@Before
- public void setUp() throws NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException{
+ public void setUp() throws NoSuchFieldException, SecurityException,
+ IllegalArgumentException, IllegalAccessException{
this.mimeTypes = TikaConfig.getDefaultConfig().getMimeRepository();
Field magicsField = mimeTypes.getClass().getDeclaredField("magics");
@@ -302,4 +305,72 @@ public class MimeTypesReaderTest {
mimeTypes.getMediaTypeRegistry().getAliases(MediaType.APPLICATION_ZIP);
}
}
+
+ @Test
+ public void testMinShouldMatch() throws Exception {
+ System.setProperty(MimeTypesFactory.CUSTOM_MIMES_SYS_PROP,
+ "src/test/resources/org/apache/tika/mime/custom-mimetypes-minShouldMatch.xml");
+ MimeTypes mimeTypes = MimeTypes.getDefaultMimeTypes(new CustomClassLoader());
+
+ //matches one
+ assertEquals("hello/world-min-file",
+ getTypeAsString(mimeTypes, "Hello World!", new Metadata()));
+
+ //matches two
+ assertEquals("hello/world-min-file",
+ getTypeAsString(mimeTypes, "Hello Welt!", new Metadata()));
+
+ //matches two
+ assertEquals("hello/world-min-file",
+ getTypeAsString(mimeTypes, "Hallo Welt!", new Metadata()));
+
+ //missing !
+ assertEquals("text/plain",
+ getTypeAsString(mimeTypes, "Hello World", new Metadata()));
+
+ //Hello requires world, welt or hallo; monde requires bonjour le
+ assertEquals("text/plain",
+ getTypeAsString(mimeTypes, "Hello Monde", new Metadata()));
+
+ //this matcher is treated as "or" with minshouldmatch clause
+ assertEquals("hello/world-min-file",
+ getTypeAsString(mimeTypes, "Bonjour le Monde!", new Metadata()));
+
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testBadMinShouldMatch1() throws Exception {
+ System.setProperty(MimeTypesFactory.CUSTOM_MIMES_SYS_PROP,
+ "src/test/resources/org/apache/tika/mime/custom-mimetypes-badMinShouldMatch1.xml");
+ MimeTypes mimeTypes = MimeTypes.getDefaultMimeTypes(new CustomClassLoader());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testBadMinShouldMatch2() throws Exception {
+ System.setProperty(MimeTypesFactory.CUSTOM_MIMES_SYS_PROP,
+ "src/test/resources/org/apache/tika/mime/custom-mimetypes-badMinShouldMatch2.xml");
+ MimeTypes mimeTypes = MimeTypes.getDefaultMimeTypes(new CustomClassLoader());
+ }
+
+ @Test(expected = NumberFormatException.class)
+ public void testBadMinShouldMatch3() throws Exception {
+ System.setProperty(MimeTypesFactory.CUSTOM_MIMES_SYS_PROP,
+ "src/test/resources/org/apache/tika/mime/custom-mimetypes-badMinShouldMatch3.xml");
+ MimeTypes mimeTypes = MimeTypes.getDefaultMimeTypes(new CustomClassLoader());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testBadMinShouldMatch4() throws Exception {
+ System.setProperty(MimeTypesFactory.CUSTOM_MIMES_SYS_PROP,
+ "src/test/resources/org/apache/tika/mime/custom-mimetypes-badMinShouldMatch4.xml");
+ MimeTypes mimeTypes = MimeTypes.getDefaultMimeTypes(new CustomClassLoader());
+ }
+
+ private static String getTypeAsString(MimeTypes mimeTypes, String text, Metadata metadata) throws IOException {
+ return mimeTypes.detect(
+ new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8)),
+ metadata
+ ).toString();
+
+ }
}
diff --git a/tika-core/src/test/resources/org/apache/tika/mime/custom-mimetypes-badMinShouldMatch1.xml b/tika-core/src/test/resources/org/apache/tika/mime/custom-mimetypes-badMinShouldMatch1.xml
new file mode 100644
index 0000000..2192538
--- /dev/null
+++ b/tika-core/src/test/resources/org/apache/tika/mime/custom-mimetypes-badMinShouldMatch1.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ Description: This test XML file is used to test minShouldMatch
+-->
+<mime-info>
+
+
+ <mime-type type="hello/world-min-file">
+ <_comment>A "Hello World MinShouldMatch" file</_comment>
+ <magic priority="50">
+ <match minShouldMatch="5">
+ <match value="Hello" type="string" offset="0:20" />
+ <match value="Hallow" type="string" offset="0:20" />
+ <match value="World!" type="string" offset="0:20" />
+ <match value="Welt!" type="string" offset="0:20" />
+ </match>
+ </magic>
+ </mime-type>
+
+
+</mime-info>
diff --git a/tika-core/src/test/resources/org/apache/tika/mime/custom-mimetypes-badMinShouldMatch2.xml b/tika-core/src/test/resources/org/apache/tika/mime/custom-mimetypes-badMinShouldMatch2.xml
new file mode 100644
index 0000000..94a1b9f
--- /dev/null
+++ b/tika-core/src/test/resources/org/apache/tika/mime/custom-mimetypes-badMinShouldMatch2.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ Description: This test XML file is used to test minShouldMatch
+-->
+<mime-info>
+
+
+ <mime-type type="hello/world-min-file">
+ <_comment>A "Hello World MinShouldMatch" file</_comment>
+ <magic priority="50">
+ <match minShouldMatch="-1">
+ <match value="Hello" type="string" offset="0:20" />
+ <match value="Hallow" type="string" offset="0:20" />
+ <match value="World!" type="string" offset="0:20" />
+ <match value="Welt!" type="string" offset="0:20" />
+ </match>
+ </magic>
+ </mime-type>
+
+
+</mime-info>
diff --git a/tika-core/src/test/resources/org/apache/tika/mime/custom-mimetypes-badMinShouldMatch3.xml b/tika-core/src/test/resources/org/apache/tika/mime/custom-mimetypes-badMinShouldMatch3.xml
new file mode 100644
index 0000000..663b2fa
--- /dev/null
+++ b/tika-core/src/test/resources/org/apache/tika/mime/custom-mimetypes-badMinShouldMatch3.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ Description: This test XML file is used to test minShouldMatch
+-->
+<mime-info>
+
+
+ <mime-type type="hello/world-min-file">
+ <_comment>A "Hello World MinShouldMatch" file</_comment>
+ <magic priority="50">
+ <match minShouldMatch="needsToBeAnInteger!">
+ <match value="Hello" type="string" offset="0:20" />
+ <match value="Hallow" type="string" offset="0:20" />
+ <match value="World!" type="string" offset="0:20" />
+ <match value="Welt!" type="string" offset="0:20" />
+ </match>
+ </magic>
+ </mime-type>
+
+
+</mime-info>
diff --git a/tika-core/src/test/resources/org/apache/tika/mime/custom-mimetypes-badMinShouldMatch4.xml b/tika-core/src/test/resources/org/apache/tika/mime/custom-mimetypes-badMinShouldMatch4.xml
new file mode 100644
index 0000000..b57fa20
--- /dev/null
+++ b/tika-core/src/test/resources/org/apache/tika/mime/custom-mimetypes-badMinShouldMatch4.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ Description: This test XML file is used to test minShouldMatch
+-->
+<mime-info>
+
+
+ <mime-type type="hello/world-min-file">
+ <_comment>A "Hello World MinShouldMatch" file</_comment>
+ <magic priority="50">
+ <match minShouldMatch="1"/>
+ </magic>
+ </mime-type>
+
+
+</mime-info>
diff --git a/tika-core/src/test/resources/org/apache/tika/mime/custom-mimetypes-minShouldMatch.xml b/tika-core/src/test/resources/org/apache/tika/mime/custom-mimetypes-minShouldMatch.xml
new file mode 100644
index 0000000..8840681
--- /dev/null
+++ b/tika-core/src/test/resources/org/apache/tika/mime/custom-mimetypes-minShouldMatch.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ Description: This test XML file is used to test minShouldMatch
+-->
+<mime-info>
+
+
+ <mime-type type="hello/world-min-file">
+ <_comment>A "Hello World MinShouldMatch" file</_comment>
+ <magic priority="50">
+ <match minShouldMatch="2">
+ <match value="Hello" type="string" offset="0:20" />
+ <match value="Hallo" type="string" offset="0:20" />
+ <match value="World!" type="string" offset="6:12" />
+ <match value="Welt!" type="string" offset="6:12" />
+ </match>
+ <match value="Bonjour le Monde!" type="string" offset="0:20" />
+ </magic>
+ </mime-type>
+
+
+</mime-info>
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
index ef8404c..6100b2d 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
@@ -289,7 +289,7 @@ public class RFC822ParserTest extends TikaTest {
"really really really really really really long name ");
}
String name = inputBuilder.toString();
- byte[] data = ("From: " + name + "\r\n\r\n").getBytes(US_ASCII);
+ byte[] data = ("Status: 520\r\nFrom: " + name + "\r\n\r\n").getBytes(US_ASCII);
ContentHandler handler = new DefaultHandler();
Metadata metadata = new Metadata();
[tika] 01/02: TIKA-3160 -- run surefire tests headless
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 811330f951cb8a0593202007c966be0325fb242e
Author: tballison <ta...@apache.org>
AuthorDate: Wed Aug 12 05:53:41 2020 -0400
TIKA-3160 -- run surefire tests headless
---
tika-parent/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tika-parent/pom.xml b/tika-parent/pom.xml
index 3d07475..c985dfc 100644
--- a/tika-parent/pom.xml
+++ b/tika-parent/pom.xml
@@ -442,7 +442,7 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>${maven.surefire.version}</version> <!-- versions greater than this don't like System.exit calls in tika-batch -->
<configuration>
- <argLine>-Xmx3072m -Duser.timezone=UTC</argLine>
+ <argLine>-Xmx3072m -Duser.timezone=UTC -Djava.awt.headless=true</argLine>
</configuration>
</plugin>
<plugin>