You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by th...@apache.org on 2014/02/17 13:16:26 UTC
svn commit: r1568954 - in /tika/trunk/tika-parsers/src:
main/java/org/apache/tika/parser/microsoft/ooxml/
test/java/org/apache/tika/parser/microsoft/ooxml/
test/resources/test-documents/
Author: thaichat04
Date: Mon Feb 17 12:16:25 2014
New Revision: 1568954
URL: http://svn.apache.org/r1568954
Log:
TIKA-1223 - Extract thumbnail of OOXML Office as attachment
Added:
tika/trunk/tika-parsers/src/test/resources/test-documents/testDOCX_Thumbnail.docx (with props)
tika/trunk/tika-parsers/src/test/resources/test-documents/testPPTX_Thumbnail.pptx (with props)
tika/trunk/tika-parsers/src/test/resources/test-documents/testXLSX_Thumbnail.xlsx (with props)
Modified:
tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLContainerExtractionTest.java
tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLParserTest.java
Modified: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java?rev=1568954&r1=1568953&r2=1568954&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java (original)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java Mon Feb 17 12:16:25 2014
@@ -16,16 +16,21 @@
*/
package org.apache.tika.parser.microsoft.ooxml;
+import static org.apache.tika.sax.XHTMLContentHandler.XHTML;
+
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.InputStream;
import java.net.URI;
import java.util.List;
import org.apache.poi.POIXMLDocument;
import org.apache.poi.POIXMLTextExtractor;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
+import org.apache.poi.openxml4j.opc.OPCPackage;
import org.apache.poi.openxml4j.opc.PackagePart;
import org.apache.poi.openxml4j.opc.PackageRelationship;
+import org.apache.poi.openxml4j.opc.PackageRelationshipTypes;
import org.apache.poi.openxml4j.opc.TargetMode;
import org.apache.poi.poifs.filesystem.DirectoryNode;
import org.apache.poi.poifs.filesystem.Ole10Native;
@@ -34,15 +39,19 @@ import org.apache.poi.poifs.filesystem.P
import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.ParsingEmbeddedDocumentExtractor;
+import org.apache.tika.io.FilenameUtils;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.microsoft.OfficeParser.POIFSDocumentType;
import org.apache.tika.sax.EmbeddedContentHandler;
import org.apache.tika.sax.XHTMLContentHandler;
import org.apache.xmlbeans.XmlException;
+import org.xml.sax.Attributes;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
+import org.xml.sax.helpers.AttributesImpl;
/**
* Base class for all Tika OOXML extractors.
@@ -106,6 +115,9 @@ public abstract class AbstractOOXMLExtra
// Now do any embedded parts
handleEmbeddedParts(handler);
+
+ // thumbnail
+ handleThumbnail(handler);
xhtml.endDocument();
}
@@ -122,6 +134,39 @@ public abstract class AbstractOOXMLExtra
return desc;
}
+
+ private void handleThumbnail( ContentHandler handler ) {
+ try {
+ OPCPackage opcPackage = extractor.getPackage();
+ int thumbIndex = 0;
+ for (PackageRelationship rel : opcPackage.getRelationshipsByType( PackageRelationshipTypes.THUMBNAIL )) {
+ PackagePart tPart = opcPackage.getPart(rel);
+ InputStream tStream = tPart.getInputStream();
+ Metadata thumbnailMetadata = new Metadata();
+ String thumbName = "thumbnail_" + thumbIndex + "." + tPart.getPartName().getExtension();
+ thumbnailMetadata.set(Metadata.RESOURCE_NAME_KEY, thumbName);
+
+ AttributesImpl attributes = new AttributesImpl();
+ attributes.addAttribute(XHTML, "class", "class", "CDATA", "embedded");
+ attributes.addAttribute(XHTML, "id", "id", "CDATA", thumbName);
+ handler.startElement(XHTML, "div", "div", attributes);
+ handler.endElement(XHTML, "div", "div");
+
+ thumbnailMetadata.set(Metadata.EMBEDDED_RELATIONSHIP_ID, thumbName);
+ thumbnailMetadata.set(Metadata.CONTENT_TYPE, tPart.getContentType());
+ thumbnailMetadata.set(TikaCoreProperties.TITLE, tPart.getPartName().getName());
+
+ if (embeddedExtractor.shouldParseEmbedded(thumbnailMetadata)) {
+ embeddedExtractor.parseEmbedded(TikaInputStream.get(tStream), new EmbeddedContentHandler(handler), thumbnailMetadata, true);
+ }
+
+ tStream.close();
+ thumbIndex ++;
+ }
+ } catch (Exception ex) {
+
+ }
+ }
private void handleEmbeddedParts(ContentHandler handler)
throws TikaException, IOException, SAXException {
Modified: tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLContainerExtractionTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLContainerExtractionTest.java?rev=1568954&r1=1568953&r2=1568954&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLContainerExtractionTest.java (original)
+++ tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLContainerExtractionTest.java Mon Feb 17 12:16:25 2014
@@ -45,22 +45,23 @@ public class OOXMLContainerExtractionTes
*/
@Test
public void testWithoutEmbedded() throws Exception {
- String[] files = new String[] {
- "testEXCEL.xlsx", "testWORD.docx", "testPPT.pptx",
- };
- for(String file : files) {
- // Process it without recursing
- TrackingHandler handler = process(file, extractor, false);
-
- // Won't have fired
- assertEquals(0, handler.filenames.size());
- assertEquals(0, handler.mediaTypes.size());
-
- // Ditto with recursing
- handler = process(file, extractor, true);
- assertEquals(0, handler.filenames.size());
- assertEquals(0, handler.mediaTypes.size());
- }
+ assertEmbeddedFiles(0, "testEXCEL.xlsx" );
+ assertEmbeddedFiles(0, "testWORD.docx" );
+ assertEmbeddedFiles(1 /* thumbnail as default */, "testPPT.pptx" );
+ }
+
+ private void assertEmbeddedFiles(int expectedNbFiles, String file ) throws Exception {
+ // Process it without recursing
+ TrackingHandler handler = process(file, extractor, false);
+
+ // Won't have fired
+ assertEquals(expectedNbFiles, handler.filenames.size());
+ assertEquals(expectedNbFiles, handler.mediaTypes.size());
+
+ // Ditto with recursing
+ handler = process(file, extractor, true);
+ assertEquals(expectedNbFiles, handler.filenames.size());
+ assertEquals(expectedNbFiles, handler.mediaTypes.size());
}
/**
@@ -83,8 +84,8 @@ public class OOXMLContainerExtractionTes
// PowerPoint with 2 images + sound
// TODO Figure out why we can't find the sound anywhere...
handler = process("testPPT_2imgs.pptx", extractor, false);
- assertEquals(3, handler.filenames.size());
- assertEquals(3, handler.mediaTypes.size());
+ assertEquals(3 + 1 /*thumbnail */, handler.filenames.size());
+ assertEquals(3 + 1 /*thumbnail */, handler.mediaTypes.size());
assertEquals("image1.png", handler.filenames.get(0));
assertEquals("image2.gif", handler.filenames.get(1));
@@ -160,32 +161,33 @@ public class OOXMLContainerExtractionTes
// With recursion, should get the images embedded in the office files too
handler = process("testEXCEL_embeded.xlsx", extractor, true);
- assertEquals(23, handler.filenames.size());
- assertEquals(23, handler.mediaTypes.size());
+ assertEquals(23 + 1 /*thumbnail */, handler.filenames.size());
+ assertEquals(23 + 1 /*thumbnail */, handler.mediaTypes.size());
assertEquals(TYPE_PPTX, handler.mediaTypes.get(0)); // Embedded office doc
- assertEquals(TYPE_PNG, handler.mediaTypes.get(1)); // PNG inside .pptx
- assertEquals(TYPE_GIF, handler.mediaTypes.get(2)); // PNG inside .pptx
- assertEquals(TYPE_PNG, handler.mediaTypes.get(3)); // PNG inside .pptx
- assertEquals(TYPE_XLSX, handler.mediaTypes.get(4)); // .xlsx inside .pptx
- assertEquals(TYPE_PNG, handler.mediaTypes.get(5)); // PNG inside .xlsx inside .pptx
- assertEquals(TYPE_DOCX, handler.mediaTypes.get(6)); // .docx inside .pptx
- assertEquals(TYPE_PNG, handler.mediaTypes.get(7)); // PNG inside .docx inside .pptx
- assertEquals(TYPE_JPG, handler.mediaTypes.get(8)); // JPG inside .docx inside .pptx
- assertEquals(TYPE_PNG, handler.mediaTypes.get(9)); // PNG inside .docx inside .pptx
- assertEquals(TYPE_DOC, handler.mediaTypes.get(10)); // .doc inside .pptx
- assertEquals(TYPE_PNG, handler.mediaTypes.get(11)); // PNG inside .doc inside .pptx
- assertEquals(TYPE_EMF, handler.mediaTypes.get(12)); // Icon of item inside .pptx
- assertEquals(TYPE_EMF, handler.mediaTypes.get(13)); // Icon of item inside .pptx
- assertEquals(TYPE_EMF, handler.mediaTypes.get(14)); // Icon of item inside .pptx
- assertEquals(TYPE_DOC, handler.mediaTypes.get(15)); // Embedded office doc
- assertEquals(TYPE_PNG, handler.mediaTypes.get(16)); // PNG inside .doc
- assertEquals(TYPE_DOCX, handler.mediaTypes.get(17)); // Embedded office doc
- assertEquals(TYPE_PNG, handler.mediaTypes.get(18)); // PNG inside .docx
- assertEquals(TYPE_PNG, handler.mediaTypes.get(19)); // Embedded image
- assertEquals(TYPE_EMF, handler.mediaTypes.get(20)); // Icon of embedded office doc
+ assertEquals(TYPE_PNG, handler.mediaTypes.get(1)); // PNG inside .pptx
+ assertEquals(TYPE_GIF, handler.mediaTypes.get(2)); // PNG inside .pptx
+ assertEquals(TYPE_PNG, handler.mediaTypes.get(3)); // PNG inside .pptx
+ assertEquals(TYPE_XLSX, handler.mediaTypes.get(4)); // .xlsx inside .pptx
+ assertEquals(TYPE_PNG, handler.mediaTypes.get(5)); // PNG inside .xlsx inside .pptx
+ assertEquals(TYPE_DOCX, handler.mediaTypes.get(6)); // .docx inside .pptx
+ assertEquals(TYPE_PNG, handler.mediaTypes.get(7)); // PNG inside .docx inside .pptx
+ assertEquals(TYPE_JPG, handler.mediaTypes.get(8)); // JPG inside .docx inside .pptx
+ assertEquals(TYPE_PNG, handler.mediaTypes.get(9)); // PNG inside .docx inside .pptx
+ assertEquals(TYPE_DOC, handler.mediaTypes.get(10)); // .doc inside .pptx
+ assertEquals(TYPE_PNG, handler.mediaTypes.get(11)); // PNG inside .doc inside .pptx
+ assertEquals(TYPE_EMF, handler.mediaTypes.get(12)); // Icon of item inside .pptx
+ assertEquals(TYPE_EMF, handler.mediaTypes.get(13)); // Icon of item inside .pptx
+ assertEquals(TYPE_EMF, handler.mediaTypes.get(14)); // Icon of item inside .pptx
+ assertEquals(TYPE_JPG, handler.mediaTypes.get(15)); // Embedded thumbnail
+ assertEquals(TYPE_DOC, handler.mediaTypes.get(16)); // Embedded office doc
+ assertEquals(TYPE_PNG, handler.mediaTypes.get(17)); // PNG inside .doc
+ assertEquals(TYPE_DOCX, handler.mediaTypes.get(18)); // Embedded office doc
+ assertEquals(TYPE_PNG, handler.mediaTypes.get(19)); // PNG inside .docx
+ assertEquals(TYPE_PNG, handler.mediaTypes.get(20)); // Embedded image
assertEquals(TYPE_EMF, handler.mediaTypes.get(21)); // Icon of embedded office doc
assertEquals(TYPE_EMF, handler.mediaTypes.get(22)); // Icon of embedded office doc
+ assertEquals(TYPE_EMF, handler.mediaTypes.get(23)); // Icon of embedded office doc
// Word with .docx, powerpoint and excel
@@ -217,30 +219,31 @@ public class OOXMLContainerExtractionTes
// With recursion, should get their images too
handler = process("testWORD_embeded.docx", extractor, true);
- assertEquals(14, handler.filenames.size());
- assertEquals(14, handler.mediaTypes.size());
+ assertEquals(14 + 1 /* thumbnail */, handler.filenames.size());
+ assertEquals(14 + 1 /* thumbnail */, handler.mediaTypes.size());
// But we do know their types
assertEquals(TYPE_PPTX, handler.mediaTypes.get(0)); // Embedded office doc
- assertEquals(TYPE_PNG, handler.mediaTypes.get(1)); // PNG inside .pptx
- assertEquals(TYPE_GIF, handler.mediaTypes.get(2)); // GIF inside .pptx
- assertEquals(TYPE_PNG, handler.mediaTypes.get(3)); // PNG inside .pptx
- assertEquals(TYPE_EMF, handler.mediaTypes.get(4)); // Icon of embedded office doc
- assertEquals(TYPE_DOC, handler.mediaTypes.get(5)); // Embedded office doc
- assertEquals(TYPE_PNG, handler.mediaTypes.get(6)); // PNG inside .doc
- assertEquals(TYPE_PNG, handler.mediaTypes.get(7)); // Embedded image
- assertEquals(TYPE_JPG, handler.mediaTypes.get(8)); // Embedded image
- assertEquals(TYPE_PNG, handler.mediaTypes.get(9)); // Embedded image
- assertEquals(TYPE_EMF, handler.mediaTypes.get(10)); // Icon of embedded office doc
- assertEquals(TYPE_XLSX, handler.mediaTypes.get(11)); // Embeded office doc
- assertEquals(TYPE_PNG, handler.mediaTypes.get(12)); // PNG inside .xlsx
- assertEquals(TYPE_EMF, handler.mediaTypes.get(13)); // Icon of embedded office doc
+ assertEquals(TYPE_PNG, handler.mediaTypes.get(1)); // PNG inside .pptx
+ assertEquals(TYPE_GIF, handler.mediaTypes.get(2)); // GIF inside .pptx
+ assertEquals(TYPE_PNG, handler.mediaTypes.get(3)); // PNG inside .pptx
+ assertEquals(TYPE_JPG, handler.mediaTypes.get(4)); // Embedded thumbnail
+ assertEquals(TYPE_EMF, handler.mediaTypes.get(5)); // Icon of embedded office doc
+ assertEquals(TYPE_DOC, handler.mediaTypes.get(6)); // Embedded office doc
+ assertEquals(TYPE_PNG, handler.mediaTypes.get(7)); // PNG inside .doc
+ assertEquals(TYPE_PNG, handler.mediaTypes.get(8)); // Embedded image
+ assertEquals(TYPE_JPG, handler.mediaTypes.get(9)); // Embedded image
+ assertEquals(TYPE_PNG, handler.mediaTypes.get(10)); // Embedded image
+ assertEquals(TYPE_EMF, handler.mediaTypes.get(11)); // Icon of embedded office doc
+ assertEquals(TYPE_XLSX, handler.mediaTypes.get(12)); // Embeded office doc
+ assertEquals(TYPE_PNG, handler.mediaTypes.get(13)); // PNG inside .xlsx
+ assertEquals(TYPE_EMF, handler.mediaTypes.get(14)); // Icon of embedded office doc
// PowerPoint with excel and word
handler = process("testPPT_embeded.pptx", extractor, false);
- assertEquals(9, handler.filenames.size());
- assertEquals(9, handler.mediaTypes.size());
+ assertEquals(9 + 1 /* thumbnail */, handler.filenames.size());
+ assertEquals(9 + 1 /* thumbnail */, handler.mediaTypes.size());
// We don't know their exact filenames
assertEquals("image4.png", handler.filenames.get(0));
Modified: tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLParserTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLParserTest.java?rev=1568954&r1=1568953&r2=1568954&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLParserTest.java (original)
+++ tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLParserTest.java Mon Feb 17 12:16:25 2014
@@ -1058,4 +1058,47 @@ public class OOXMLParserTest extends Tik
"<p>2011-12-19 10:20:04 AM</p>\n", result.xml);
}
+
+ //TIKA-1223
+ @Test
+ public void testDOCXThumbnail() throws Exception {
+ String xml = getXML("testDOCX_Thumbnail.docx").xml;
+ int a = xml.indexOf("This file contains a thumbnail");
+ int b = xml.indexOf("<div class=\"embedded\" id=\"thumbnail_0.emf\" />");
+ int c = xml.indexOf( "<div class=\"package-entry\"><h1>thumbnail_0.emf</h1></div>" );
+
+ assertTrue(a != -1);
+ assertTrue(b != -1);
+ assertTrue(c != -1);
+ assertTrue(a < b);
+ assertTrue(b < c);
+ }
+
+ @Test
+ public void testXLSXThumbnail() throws Exception {
+ String xml = getXML("testXLSX_Thumbnail.xlsx").xml;
+ int a = xml.indexOf("This file contains an embedded thumbnail by default");
+ int b = xml.indexOf("<div class=\"embedded\" id=\"thumbnail_0.wmf\" />");
+ int c = xml.indexOf( "<div class=\"package-entry\"><h1>thumbnail_0.wmf</h1></div>" );
+
+ assertTrue(a != -1);
+ assertTrue(b != -1);
+ assertTrue(c != -1);
+ assertTrue(a < b);
+ assertTrue(b < c);
+ }
+
+ @Test
+ public void testPPTXThumbnail() throws Exception {
+ String xml = getXML("testPPTX_Thumbnail.pptx").xml;
+ int a = xml.indexOf("<body><p>This file contains an embedded thumbnail</p>");
+ int b = xml.indexOf("<div class=\"embedded\" id=\"thumbnail_0.jpeg\" />");
+ int c = xml.indexOf( "<div class=\"package-entry\"><h1>thumbnail_0.jpeg</h1></div>" );
+
+ assertTrue(a != -1);
+ assertTrue(b != -1);
+ assertTrue(c != -1);
+ assertTrue(a < b);
+ assertTrue(b < c);
+ }
}
Added: tika/trunk/tika-parsers/src/test/resources/test-documents/testDOCX_Thumbnail.docx
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/resources/test-documents/testDOCX_Thumbnail.docx?rev=1568954&view=auto
==============================================================================
Binary file - no diff available.
Propchange: tika/trunk/tika-parsers/src/test/resources/test-documents/testDOCX_Thumbnail.docx
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: tika/trunk/tika-parsers/src/test/resources/test-documents/testPPTX_Thumbnail.pptx
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/resources/test-documents/testPPTX_Thumbnail.pptx?rev=1568954&view=auto
==============================================================================
Binary file - no diff available.
Propchange: tika/trunk/tika-parsers/src/test/resources/test-documents/testPPTX_Thumbnail.pptx
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: tika/trunk/tika-parsers/src/test/resources/test-documents/testXLSX_Thumbnail.xlsx
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/resources/test-documents/testXLSX_Thumbnail.xlsx?rev=1568954&view=auto
==============================================================================
Binary file - no diff available.
Propchange: tika/trunk/tika-parsers/src/test/resources/test-documents/testXLSX_Thumbnail.xlsx
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream