You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2021/12/03 23:25:46 UTC
[tika] 01/02: TIKA-3526 -- improve extraction of attachments from MSOffice files created by non-MS tools
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit cace0ca11330588c21e67476820d4f920034d99a
Author: tallison <ta...@apache.org>
AuthorDate: Fri Dec 3 18:24:32 2021 -0500
TIKA-3526 -- improve extraction of attachments from MSOffice files created by non-MS tools
---
CHANGES.txt | 3 ++
.../detect/microsoft/ooxml/OPCPackageDetector.java | 12 +++++-
.../parser/microsoft/AbstractPOIFSExtractor.java | 8 ++--
.../tika/parser/microsoft/HSLFExtractor.java | 44 ++++++++++++++++++-
.../microsoft/ooxml/AbstractOOXMLExtractor.java | 44 +++++++++++++------
.../microsoft/POIContainerExtractionTest.java | 48 +++++++++++++++++++--
.../resources/test-documents/testWPSAttachment.doc | Bin 0 -> 222720 bytes
.../test-documents/testWPSAttachment.docx | Bin 0 -> 130189 bytes
.../resources/test-documents/testWPSAttachment.ppt | Bin 0 -> 269312 bytes
.../test-documents/testWPSAttachment.pptx | Bin 0 -> 151705 bytes
.../resources/test-documents/testWPSAttachment.xls | Bin 0 -> 229888 bytes
.../test-documents/testWPSAttachment.xlsx | Bin 0 -> 131498 bytes
.../microsoft/POIContainerExtractionTest.java | 1 -
13 files changed, 135 insertions(+), 25 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index f3bcdf2..7348acc 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,8 @@
Release 2.2.0 - ???
+ * Improve extraction of embedded files from MSOffice files created
+ by non-Microsoft tools (TIKA-3526).
+
* Added back ability to ignore load errors in TikaConfig (TIKA-3575).
* Make SecureContentHandler and other parameters configurable in
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/ooxml/OPCPackageDetector.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/ooxml/OPCPackageDetector.java
index 64a39e2..dd58d10 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/ooxml/OPCPackageDetector.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/ooxml/OPCPackageDetector.java
@@ -282,15 +282,23 @@ public class OPCPackageDetector implements ZipContainerDetector {
detectContext.set(OOXMLHintCounter.class, cnt);
}
cnt.increment();
- if (cnt.getCount() > 2) {
+ //we used to short-circuit, but ooxml files that are
+ //not created by actual microsoft products do not tend to
+ //put [Content_Types.xml] as the first physical entry in the zip file
+ //so we need to read through the full file
+ /*if (cnt.getCount() > 2) {
return TIKA_OOXML;
- }
+ }*/
}
return null;
}
@Override
public MediaType streamingDetectFinal(StreamingDetectContext detectContext) {
+ OOXMLHintCounter cnt = detectContext.get(OOXMLHintCounter.class);
+ if (cnt != null && cnt.getCount() > 2) {
+ return TIKA_OOXML;
+ }
return null;
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java
index 19cc622..f4ee3e8 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java
@@ -157,12 +157,14 @@ abstract class AbstractPOIFSExtractor {
XHTMLContentHandler xhtml)
throws IOException, SAXException, TikaException {
+
// Is it an embedded OLE2 document, or an embedded OOXML document?
+ //first try for ooxml
+ Entry ooxml = dir.hasEntry("Package") ? dir.getEntry("Package") :
+ (dir.hasEntry("package") ? dir.getEntry("package") : null);
- if (dir.hasEntry("Package")) {
+ if (ooxml != null) {
// It's OOXML (has a ZipFile):
- Entry ooxml = dir.getEntry("Package");
-
try (TikaInputStream stream = TikaInputStream
.get(new DocumentInputStream((DocumentEntry) ooxml))) {
Detector detector = new DefaultZipContainerDetector();
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/HSLFExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/HSLFExtractor.java
index 2b20089..38de8f2 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/HSLFExtractor.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/HSLFExtractor.java
@@ -19,7 +19,9 @@ package org.apache.tika.parser.microsoft;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.apache.poi.common.usermodel.Hyperlink;
@@ -44,6 +46,7 @@ import org.apache.poi.hslf.usermodel.HSLFTextParagraph;
import org.apache.poi.hslf.usermodel.HSLFTextRun;
import org.apache.poi.hslf.usermodel.HSLFTextShape;
import org.apache.poi.poifs.filesystem.DirectoryNode;
+import org.apache.poi.poifs.filesystem.FileMagic;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
import org.apache.poi.sl.usermodel.Comment;
import org.apache.poi.sl.usermodel.ShapeContainer;
@@ -65,6 +68,7 @@ public class HSLFExtractor extends AbstractPOIFSExtractor {
public HSLFExtractor(ParseContext context, Metadata metadata) {
super(context, metadata);
}
+ private final Set<Integer> processedEmbeddedObjects = new HashSet<>();
// remove trailing paragraph break
private static String removePBreak(String fragment) {
@@ -150,6 +154,8 @@ public class HSLFExtractor extends AbstractPOIFSExtractor {
}
handleSlideEmbeddedPictures(ss, xhtml);
+ handleShowEmbeddedResources(ss, xhtml);
+
if (officeParserConfig.isExtractMacros()) {
extractMacros(ss, xhtml);
}
@@ -160,6 +166,39 @@ public class HSLFExtractor extends AbstractPOIFSExtractor {
xhtml.endElement("div");
}
+ /**
+ * This is the catch-all for embedded objects. If we didn't come across
+ * them in the shapes in the slides, headers/footers, etc, try to
+ * extract them here.
+ **/
+ private void handleShowEmbeddedResources(HSLFSlideShow ss, XHTMLContentHandler xhtml)
+ throws SAXException {
+
+ HSLFObjectData[] objectData = ss.getEmbeddedObjects();
+
+ for (HSLFObjectData d : objectData) {
+ if (processedEmbeddedObjects.contains(d.getExOleObjStg().getPersistId())) {
+ continue;
+ }
+ try (TikaInputStream tis = TikaInputStream.get(d.getInputStream())) {
+ if (FileMagic.valueOf(tis) == FileMagic.OLE2) {
+ try (POIFSFileSystem pfs = new POIFSFileSystem(tis)) {
+ //coz ppts can have empty pfs...shrug...
+ //we may need to add more stringent requirements
+ if (pfs.getRoot().getEntryNames().size() < 1) {
+ return;
+ }
+ handleEmbeddedOfficeDoc(pfs.getRoot(), d.getFileName(), xhtml);
+ }
+ } else {
+ handleEmbeddedResource(tis, d.getFileName(), null, null, xhtml, true);
+ }
+ } catch (IOException | TikaException e) {
+ EmbeddedDocumentUtil.recordException(e, parentMetadata);
+ }
+ }
+ }
+
private void handleComments(HSLFSlide slide, XHTMLContentHandler xhtml) throws SAXException {
if (slide.getComments() == null || slide.getComments().size() == 0) {
return;
@@ -469,7 +508,7 @@ public class HSLFExtractor extends AbstractPOIFSExtractor {
private void handleSlideEmbeddedResources(ShapeContainer shapeContainer,
XHTMLContentHandler xhtml)
- throws TikaException, SAXException, IOException {
+ throws TikaException, SAXException {
List<HSLFShape> shapes = getShapes(shapeContainer);
if (shapes == null) {
return;
@@ -478,6 +517,7 @@ public class HSLFExtractor extends AbstractPOIFSExtractor {
//handle ActiveXShape, movie shape?
if (shape instanceof HSLFObjectShape) {
HSLFObjectShape oleShape = (HSLFObjectShape) shape;
+
HSLFObjectData data = null;
try {
data = oleShape.getObjectData();
@@ -489,7 +529,7 @@ public class HSLFExtractor extends AbstractPOIFSExtractor {
if (data != null) {
String objID = Integer.toString(oleShape.getObjectID());
-
+ processedEmbeddedObjects.add(data.getExOleObjStg().getPersistId());
// Embedded Object: add a <div
// class="embedded" id="X"/> so consumer can see where
// in the main text each embedded document
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
index f23ae12..e1a6598 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
@@ -242,6 +242,7 @@ public abstract class AbstractOOXMLExtractor implements OOXMLExtractor {
return;
}
}
+
URI sourceURI = rel.getSourceURI();
String sourceDesc;
if (sourceURI != null) {
@@ -269,16 +270,22 @@ public abstract class AbstractOOXMLExtractor implements OOXMLExtractor {
if (POIXMLDocument.OLE_OBJECT_REL_TYPE.equals(type) &&
TYPE_OLE_OBJECT.equals(target.getContentType())) {
handleEmbeddedOLE(target, handler, sourceDesc + rel.getId(), parentMetadata);
- handledTarget.add(targetURI.toString());
+ if (targetURI != null) {
+ handledTarget.add(targetURI.toString());
+ }
} else if (RELATION_MEDIA.equals(type) || RELATION_VIDEO.equals(type) ||
RELATION_AUDIO.equals(type) || PackageRelationshipTypes.IMAGE_PART.equals(type) ||
POIXMLDocument.PACK_OBJECT_REL_TYPE.equals(type) ||
POIXMLDocument.OLE_OBJECT_REL_TYPE.equals(type)) {
handleEmbeddedFile(target, handler, sourceDesc + rel.getId());
- handledTarget.add(targetURI.toString());
+ if (targetURI != null) {
+ handledTarget.add(targetURI.toString());
+ }
} else if (XSSFRelation.VBA_MACROS.getRelation().equals(type)) {
handleMacros(target, handler);
- handledTarget.add(targetURI.toString());
+ if (targetURI != null) {
+ handledTarget.add(targetURI.toString());
+ }
}
}
@@ -311,21 +318,13 @@ public abstract class AbstractOOXMLExtractor implements OOXMLExtractor {
DirectoryNode root = fs.getRoot();
POIFSDocumentType type = POIFSDocumentType.detectType(root);
- if (root.hasEntry("\u0001Ole") && root.hasEntry("\u0001CompObj") &&
- (root.hasEntry("CONTENTS") || root.hasEntry("Package"))) {
+ String packageEntryName = getPackageEntryName(root);
+ if (packageEntryName != null) {
// TIKA-704: OLE 2.0 embedded non-Office document?
//TODO: figure out if the equivalent of OLE 1.0's
//getCommand() and getFileName() exist for OLE 2.0 to populate
//TikaCoreProperties.ORIGINAL_RESOURCE_NAME
- if (root.hasEntry("CONTENTS")) {
- stream = TikaInputStream.get(fs.createDocumentInputStream("CONTENTS"));
- } else if (root.hasEntry("Package")) {
- //TIKA-2588
- stream = TikaInputStream.get(fs.createDocumentInputStream("Package"));
- } else {
- throw new IllegalStateException(
- "Shouldn't ever arrive here; please open a ticket on our jira");
- }
+ stream = TikaInputStream.get(fs.createDocumentInputStream(packageEntryName));
if (embeddedExtractor.shouldParseEmbedded(metadata)) {
embeddedExtractor
.parseEmbedded(stream, new EmbeddedContentHandler(handler), metadata,
@@ -372,6 +371,23 @@ public abstract class AbstractOOXMLExtractor implements OOXMLExtractor {
}
}
+ private String getPackageEntryName(DirectoryNode root) {
+ if (root.hasEntry("\u0001Ole")) {
+ //we used to require this too: root.hasEntry("\u0001CompObj") before TIKA-3526
+ if (root.hasEntry("Package")) {
+ return "Package";
+ } else if (root.hasEntry("CONTENTS")) {
+ return "CONTENTS";
+ } else if (root.hasEntry("package")) {
+ return "package";
+ }
+ }
+ if (root.hasEntry("package")) {
+ return "package";
+ }
+ return null;
+ }
+
/**
* Handles an embedded file in the document
*/
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java
index 55ec414..203bdac 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java
@@ -19,8 +19,11 @@ package org.apache.tika.parser.microsoft;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import org.junit.jupiter.api.Test;
@@ -43,7 +46,7 @@ public class POIContainerExtractionTest extends AbstractPOIContainerExtractionTe
@Test
public void testWithoutEmbedded() throws Exception {
ContainerExtractor extractor = new ParserContainerExtractor();
-
+ debug(getRecursiveMetadata("testPPT.ppt"));
String[] files =
new String[]{"testEXCEL.xls", "testWORD.doc", "testPPT.ppt", "testVISIO.vsd",
"test-outlook.msg"};
@@ -52,8 +55,8 @@ public class POIContainerExtractionTest extends AbstractPOIContainerExtractionTe
TrackingHandler handler = process(file, extractor, false);
// Won't have fired
- assertEquals(0, handler.filenames.size());
- assertEquals(0, handler.mediaTypes.size());
+ assertEquals(0, handler.filenames.size(), file);
+ assertEquals(0, handler.mediaTypes.size(), file);
// Ditto with recursing
handler = process(file, extractor, true);
@@ -168,4 +171,43 @@ public class POIContainerExtractionTest extends AbstractPOIContainerExtractionTe
assertEquals(3, metadataList.size());
assertEquals("application/vnd.ms-equation", metadataList.get(2).get(Metadata.CONTENT_TYPE));
}
+
+ @Test
+ public void testWPSVariantAttachments() throws Exception {
+ //test that files created by WPS have embedded files extracted
+ //TIKA-3526
+ Set<String> expected = new HashSet<>();
+ expected.add("application/msword");
+ expected.add("application/vnd.ms-excel");
+ expected.add("application/vnd.ms-powerpoint");
+ expected.add("application/vnd.openxmlformats-officedocument.wordprocessingml.document");
+ expected.add("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
+ expected.add("application/vnd.openxmlformats-officedocument.presentationml.presentation");
+ expected.add("application/pdf");
+ expected.add("application/xml");
+ expected.add("text/plain; charset=ISO-8859-1");
+ //test that we're correctly handling attachment variants for
+ // files created by WPS 表格 (https://www.wps.cn/)
+ for (String suffix : new String[]{
+ "ppt", "doc", "xls", "docx", "pptx", "xlsx"
+ }) {
+ List<Metadata> metadataList = getRecursiveMetadata("testWPSAttachment." + suffix);
+ Set<String> found = new HashSet<>();
+ int i = 0;
+ for (Metadata m : metadataList) {
+ if (i++ > 0) {
+ found.add(m.get(Metadata.CONTENT_TYPE));
+ }
+ }
+ Set<String> notFound = new HashSet<>();
+ for (String ex : expected) {
+ if (! found.contains(ex)) {
+ notFound.add(ex);
+ }
+ }
+ if (notFound.size() > 0) {
+ fail("Couldn't find: " + notFound);
+ }
+ }
+ }
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.doc b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.doc
new file mode 100644
index 0000000..6958362
Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.doc differ
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.docx b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.docx
new file mode 100644
index 0000000..115bfa4
Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.docx differ
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.ppt b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.ppt
new file mode 100644
index 0000000..e75b17d
Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.ppt differ
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.pptx b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.pptx
new file mode 100644
index 0000000..2a5c9c8
Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.pptx differ
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.xls b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.xls
new file mode 100644
index 0000000..27a451b
Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.xls differ
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.xlsx b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.xlsx
new file mode 100644
index 0000000..38f1c28
Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.xlsx differ
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java
index 6fb166e..783554f 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java
@@ -45,7 +45,6 @@ public class POIContainerExtractionTest extends AbstractPOIContainerExtractionTe
ContainerExtractor extractor = new ParserContainerExtractor();
TikaTest.TrackingHandler handler;
-
// Excel with a word doc and a powerpoint doc, both of which have images in them
// Without recursion, should see both documents + the images
handler = process("testEXCEL_embeded.xls", extractor, false);