You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2019/06/04 16:31:27 UTC
[tika] branch master updated: TIKA-2886 -- improving streaming zip
container detection for xltx, xltm and several others.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/master by this push:
new b1adfb9 TIKA-2886 -- improving streaming zip container detection for xltx, xltm and several others.
b1adfb9 is described below
commit b1adfb9c67e6321264f4451af822f999081fc326
Author: TALLISON <ta...@apache.org>
AuthorDate: Tue Jun 4 12:30:32 2019 -0400
TIKA-2886 -- improving streaming zip container detection for xltx,
xltm and several others.
---
CHANGES.txt | 6 ++++-
.../parser/pkg/StreamingZipContainerDetector.java | 28 ++++++++++++++++++---
.../tika/parser/pkg/ZipContainerDetector.java | 5 +++-
.../tika/parser/pkg/ZipContainerDetectorBase.java | 6 +++++
.../tika/detect/TestContainerAwareDetector.java | 20 ++++++++++++++-
.../tika/parser/pkg/ZipContainerDetectorTest.java | 3 ++-
.../testEXCEL_macro_enabled_template.xltm | Bin 0 -> 8619 bytes
.../test-documents/testEXCEL_template.xlt | Bin 0 -> 25600 bytes
.../test-documents/testEXCEL_template.xltx | Bin 0 -> 8589 bytes
9 files changed, 61 insertions(+), 7 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 04794bb..369cdcb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -5,8 +5,12 @@ Release 2.0.0 - ???
Other changes
+Release 1.22 - ???
-Release 1.21 - ????
+ * Improve StreamingZipContainerDetector for xltx, xltm and
+ several other file formats (TIKA-2886).
+
+Release 1.21 - 05/14/2019
* Add optional AUTO mode to OCR'ing of PDFs. If tesseract is installed
and on the path, and this option is selected programmatically
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/pkg/StreamingZipContainerDetector.java b/tika-parsers/src/main/java/org/apache/tika/parser/pkg/StreamingZipContainerDetector.java
index 61db730..a9d135c 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/pkg/StreamingZipContainerDetector.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/pkg/StreamingZipContainerDetector.java
@@ -27,10 +27,13 @@ import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.io.IOUtils;
+import org.apache.poi.xdgf.usermodel.XDGFRelation;
import org.apache.poi.xslf.usermodel.XSLFRelation;
import org.apache.poi.xssf.usermodel.XSSFRelation;
import org.apache.poi.xwpf.usermodel.XWPFRelation;
+import org.apache.tika.detect.Detector;
import org.apache.tika.io.CloseShieldInputStream;
+import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.iwork.IWorkPackageParser;
@@ -39,17 +42,21 @@ import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
-class StreamingZipContainerDetector extends ZipContainerDetectorBase {
+public class StreamingZipContainerDetector extends ZipContainerDetectorBase implements Detector {
static Map<String, MediaType> OOXML_CONTENT_TYPES = new ConcurrentHashMap<>();
static {
OOXML_CONTENT_TYPES.put(XWPFRelation.DOCUMENT.getContentType(), DOCX);
OOXML_CONTENT_TYPES.put(XWPFRelation.MACRO_DOCUMENT.getContentType(), DOCM);
+ OOXML_CONTENT_TYPES.put(XWPFRelation.MACRO_TEMPLATE_DOCUMENT.getContentType(), DOTM);
OOXML_CONTENT_TYPES.put(XWPFRelation.TEMPLATE.getContentType(), DOTX);
OOXML_CONTENT_TYPES.put(XSSFRelation.WORKBOOK.getContentType(), XLSX);
OOXML_CONTENT_TYPES.put(XSSFRelation.MACROS_WORKBOOK.getContentType(), XLSM);
OOXML_CONTENT_TYPES.put(XSSFRelation.XLSB_BINARY_WORKBOOK.getContentType(), XLSB);
+ OOXML_CONTENT_TYPES.put(XSSFRelation.TEMPLATE_WORKBOOK.getContentType(), XLTX);
+ OOXML_CONTENT_TYPES.put(XSSFRelation.MACRO_TEMPLATE_WORKBOOK.getContentType(), XLTM);
+
OOXML_CONTENT_TYPES.put(XSLFRelation.MAIN.getContentType(), PPTX);
OOXML_CONTENT_TYPES.put(XSLFRelation.MACRO.getContentType(), PPSM);
OOXML_CONTENT_TYPES.put(XSLFRelation.MACRO_TEMPLATE.getContentType(), POTM);
@@ -58,7 +65,21 @@ class StreamingZipContainerDetector extends ZipContainerDetectorBase {
OOXML_CONTENT_TYPES.put(XSLFRelation.PRESENTATION_MACRO.getContentType(), PPTM);
OOXML_CONTENT_TYPES.put(XSLFRelation.PRESENTATIONML_TEMPLATE.getContentType(), POTX);
OOXML_CONTENT_TYPES.put(XSLFRelation.THEME_MANAGER.getContentType(), THMX);
+
+ OOXML_CONTENT_TYPES.put("application/vnd.ms-visio.drawing.macroEnabled.main+xml",
+ MediaType.application("vnd.ms-visio.drawing.macroEnabled.12"));
+ OOXML_CONTENT_TYPES.put(XDGFRelation.DOCUMENT.getContentType(), MediaType.application("vnd.ms-visio.drawing"));
+ OOXML_CONTENT_TYPES.put("application/vnd.ms-visio.stencil.macroEnabled.main+xml",
+ MediaType.application("vnd.ms-visio.stencil.macroenabled.12"));
+ OOXML_CONTENT_TYPES.put("application/vnd.ms-visio.stencil.main+xml",
+ MediaType.application("vnd.ms-visio.stencil"));
+ OOXML_CONTENT_TYPES.put("application/vnd.ms-visio.template.macroEnabled.main+xml",
+ MediaType.application("vnd.ms-visio.template.macroenabled.12"));
+ OOXML_CONTENT_TYPES.put("application/vnd.ms-visio.template.main+xml",
+ MediaType.application("vnd.ms-visio.template"));
+
OOXML_CONTENT_TYPES.put("application/vnd.ms-package.xps-fixeddocumentsequence+xml", XPS);
+
}
/**
@@ -70,7 +91,8 @@ class StreamingZipContainerDetector extends ZipContainerDetectorBase {
* if you want to protect against reading the entire stream.
* @return
*/
- static MediaType detect(InputStream is) {
+ @Override
+ public MediaType detect(InputStream is, Metadata metadata) {
Set<String> fileNames = new HashSet<>();
Set<String> directoryNames = new HashSet<>();
@@ -114,7 +136,7 @@ class StreamingZipContainerDetector extends ZipContainerDetectorBase {
}
//entrynames is the union of directory names and file names
Set<String> entryNames = new HashSet<>(fileNames);
- entryNames.addAll(fileNames);
+ entryNames.addAll(directoryNames);
MediaType mt = detectKmz(fileNames);
if (mt != null) {
return mt;
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ZipContainerDetector.java b/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ZipContainerDetector.java
index 7007b7c6..d35668c 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ZipContainerDetector.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ZipContainerDetector.java
@@ -93,6 +93,9 @@ public class ZipContainerDetector implements Detector {
@Field
int markLimit = 16 * 1024 * 1024;
+ private StreamingZipContainerDetector streamingZipContainerDetector = new StreamingZipContainerDetector();
+
+ @Override
public MediaType detect(InputStream input, Metadata metadata)
throws IOException {
// Check if we have access to the document
@@ -126,7 +129,7 @@ public class ZipContainerDetector implements Detector {
}
try (LookaheadInputStream lookahead = new LookaheadInputStream(input, markLimit)) {
- return StreamingZipContainerDetector.detect(lookahead);
+ return streamingZipContainerDetector.detect(lookahead, metadata);
}
} else if (!type.equals(MediaType.OCTET_STREAM)) {
return type;
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ZipContainerDetectorBase.java b/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ZipContainerDetectorBase.java
index 1e3aa58..5595c42 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ZipContainerDetectorBase.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ZipContainerDetectorBase.java
@@ -32,6 +32,8 @@ abstract class ZipContainerDetectorBase {
MediaType.application("vnd.openxmlformats-officedocument.wordprocessingml.document");
static final MediaType DOCM =
MediaType.application("vnd.ms-word.document.macroEnabled.12");
+ static final MediaType DOTM =
+ MediaType.application("vnd.ms-word.template.macroenabled.12");
static final MediaType DOTX =
MediaType.application("vnd.ms-word.document.macroEnabled.12");
static final MediaType PPTX =
@@ -54,6 +56,10 @@ abstract class ZipContainerDetectorBase {
MediaType.application("vnd.openxmlformats-officedocument.spreadsheetml.sheet");
static final MediaType XLSM =
MediaType.application("vnd.ms-excel.sheet.macroEnabled.12");
+ static final MediaType XLTM =
+ MediaType.application("vnd.ms-excel.template.macroenabled.12");
+ static final MediaType XLTX =
+ MediaType.application("vnd.openxmlformats-officedocument.spreadsheetml.template");
static final MediaType XPS =
MediaType.application("vnd.ms-xpsdocument");
diff --git a/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java b/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java
index 77df753..4548cb4 100644
--- a/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java
+++ b/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java
@@ -36,8 +36,10 @@ import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
+import org.apache.tika.mime.MediaTypeRegistry;
import org.apache.tika.mime.MimeTypes;
import org.apache.tika.parser.iwork.iwana.IWork13PackageParser;
+import org.apache.tika.parser.pkg.StreamingZipContainerDetector;
import org.apache.tika.utils.XMLReaderUtils;
import org.junit.After;
import org.junit.Test;
@@ -48,7 +50,9 @@ import org.junit.Test;
public class TestContainerAwareDetector extends MultiThreadedTikaTest {
private final TikaConfig tikaConfig = TikaConfig.getDefaultConfig();
private final MimeTypes mimeTypes = tikaConfig.getMimeRepository();
+ private final MediaTypeRegistry mediaTypeRegistry = mimeTypes.getMediaTypeRegistry();
private final Detector detector = new DefaultDetector(mimeTypes);
+ private final StreamingZipContainerDetector streamingZipDetector = new StreamingZipContainerDetector();
@After
public void tearDown() throws TikaException {
@@ -83,10 +87,21 @@ public class TestContainerAwareDetector extends MultiThreadedTikaTest {
mimeTypes.detect(stream, m));
}
+ MediaType expected = MediaType.parse(typeFromDetector);
// All being well, the detector should get it perfect
assertEquals(
- MediaType.parse(typeFromDetector),
+ expected,
detector.detect(stream, m));
+
+ if (mediaTypeRegistry.isSpecializationOf(
+ expected, MediaType.APPLICATION_ZIP) &&
+ ! expected.toString().contains("tika-ooxml-protected")) {
+
+ assertEquals(
+ expected,
+ streamingZipDetector.detect(stream, m));
+ }
+
}
}
@@ -223,6 +238,9 @@ public class TestContainerAwareDetector extends MultiThreadedTikaTest {
assertTypeByData("testPPT.ppsm", "application/vnd.ms-powerpoint.slideshow.macroEnabled.12");
assertTypeByData("testDOTM.dotm", "application/vnd.ms-word.template.macroEnabled.12");
assertTypeByData("testEXCEL.strict.xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
+ assertTypeByData("testEXCEL_macro_enabled_template.xltm", "application/vnd.ms-excel.template.macroenabled.12");
+ assertTypeByData("testEXCEL_template.xltx", "application/vnd.openxmlformats-officedocument.spreadsheetml.template");
+
assertTypeByData("testPPT.xps", "application/vnd.ms-xpsdocument");
assertTypeByData("testVISIO.vsdm", "application/vnd.ms-visio.drawing.macroenabled.12");
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/pkg/ZipContainerDetectorTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/pkg/ZipContainerDetectorTest.java
index 0e3ff97..d05c41a 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/pkg/ZipContainerDetectorTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/pkg/ZipContainerDetectorTest.java
@@ -46,6 +46,7 @@ public class ZipContainerDetectorTest extends TikaTest {
private static MediaType ODT_TEXT = MediaType.application("vnd.oasis.opendocument.text");
private static MediaType TIFF = MediaType.image("tiff");
ZipContainerDetector zipContainerDetector = new ZipContainerDetector();
+ StreamingZipContainerDetector streamingZipDetector = new StreamingZipContainerDetector();
@Test
public void testTiffWorkaround() throws Exception {
@@ -99,7 +100,7 @@ public class ZipContainerDetectorTest extends TikaTest {
long start = System.currentTimeMillis();
try (InputStream input = ODFParserTest.class.getResourceAsStream(
"/test-documents/" + file)) {
- MediaType mediaType = StreamingZipContainerDetector.detect(input);
+ MediaType mediaType = streamingZipDetector.detect(input, new Metadata());
assertEquals(ZipContainerDetectorBase.XPS, mediaType);
}
try (TikaInputStream input = TikaInputStream.get(Paths.get(ODFParserTest.class.getResource(
diff --git a/tika-parsers/src/test/resources/test-documents/testEXCEL_macro_enabled_template.xltm b/tika-parsers/src/test/resources/test-documents/testEXCEL_macro_enabled_template.xltm
new file mode 100644
index 0000000..1e76f58
Binary files /dev/null and b/tika-parsers/src/test/resources/test-documents/testEXCEL_macro_enabled_template.xltm differ
diff --git a/tika-parsers/src/test/resources/test-documents/testEXCEL_template.xlt b/tika-parsers/src/test/resources/test-documents/testEXCEL_template.xlt
new file mode 100644
index 0000000..81e2b18
Binary files /dev/null and b/tika-parsers/src/test/resources/test-documents/testEXCEL_template.xlt differ
diff --git a/tika-parsers/src/test/resources/test-documents/testEXCEL_template.xltx b/tika-parsers/src/test/resources/test-documents/testEXCEL_template.xltx
new file mode 100644
index 0000000..7bf75a2
Binary files /dev/null and b/tika-parsers/src/test/resources/test-documents/testEXCEL_template.xltx differ