You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2016/04/05 17:15:52 UTC
tika git commit: TIKA-1033 -- add identification for embedded
MSChart.Graph files.
Repository: tika
Updated Branches:
refs/heads/2.x 33b9e483d -> 862234289
TIKA-1033 -- add identification for embedded MSChart.Graph files.
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/86223428
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/86223428
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/86223428
Branch: refs/heads/2.x
Commit: 862234289514dede8362c04f64305a47b0580ec8
Parents: 33b9e48
Author: tballison <ta...@mitre.org>
Authored: Tue Apr 5 11:15:42 2016 -0400
Committer: tballison <ta...@mitre.org>
Committed: Tue Apr 5 11:15:42 2016 -0400
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../src/test/java/org/apache/tika/TikaTest.java | 25 +++++++++++++
.../tika/parser/microsoft/OfficeParser.java | 3 +-
.../microsoft/POIFSContainerDetector.java | 21 ++++++++++-
.../AbstractPOIContainerExtractionTest.java | 4 +--
.../microsoft/POIContainerExtractionTest.java | 35 ++++++++++---------
.../ooxml/OOXMLContainerExtractionTest.java | 23 +++++++++++-
.../testMSChart-govdocs-428996.ppt | Bin 0 -> 41472 bytes
.../testMSChart-govdocs-428996.pptx | Bin 0 -> 56224 bytes
.../testMSChart-govdocs-428996.xls | Bin 0 -> 35328 bytes
.../testMSChart-govdocs-428996.xlsx | Bin 0 -> 17112 bytes
11 files changed, 91 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index bc9db91..8459de1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -10,6 +10,8 @@ Release 2.0 - Future Development
Release 1.13 - ???
+ * Add detection for embedded MSChart.Graph files (TIKA-1033).
+
* Fix NPE in Sqlite parser from Nick C (TIKA-1927).
* Fix NPE in Open Document parser from Nick C (TIKA-1916).
http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-core/src/test/java/org/apache/tika/TikaTest.java
----------------------------------------------------------------------
diff --git a/tika-core/src/test/java/org/apache/tika/TikaTest.java b/tika-core/src/test/java/org/apache/tika/TikaTest.java
index 77215d4..fecfdf6 100644
--- a/tika-core/src/test/java/org/apache/tika/TikaTest.java
+++ b/tika-core/src/test/java/org/apache/tika/TikaTest.java
@@ -43,9 +43,12 @@ import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.RecursiveParserWrapper;
+import org.apache.tika.sax.BasicContentHandlerFactory;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.sax.ToXMLContentHandler;
import org.xml.sax.ContentHandler;
+import org.xml.sax.helpers.DefaultHandler;
/**
* Parent class of Tika tests
@@ -194,6 +197,16 @@ public abstract class TikaTest {
return getText(is, parser, new ParseContext(), new Metadata());
}
+ protected List<Metadata> getRecursiveJson(String filePath) throws Exception {
+ Parser p = new AutoDetectParser();
+ RecursiveParserWrapper wrapper = new RecursiveParserWrapper(p,
+ new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.XML, -1));
+ try (InputStream is = getResourceAsStream("/test-documents/" + filePath)) {
+ wrapper.parse(is, new DefaultHandler(), new Metadata(), new ParseContext());
+ }
+ return wrapper.getMetadata();
+ }
+
/**
* Keeps track of media types and file names recursively.
*
@@ -247,4 +260,16 @@ public abstract class TikaTest {
}
}
}
+
+ public static void debug(List<Metadata> list) {
+ int i = 0;
+ for (Metadata m : list) {
+ for (String n : m.names()) {
+ for (String v : m.getValues(n)) {
+ System.out.println(i + ": "+n + " : "+v);
+ }
+ }
+ i++;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java
index 9dd0125..84ce39e 100644
--- a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java
+++ b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java
@@ -229,7 +229,8 @@ public class OfficeParser extends AbstractParser {
OUTLOOK("msg", MediaType.application("vnd.ms-outlook")),
SOLIDWORKS_PART("sldprt", MediaType.application("sldworks")),
SOLIDWORKS_ASSEMBLY("sldasm", MediaType.application("sldworks")),
- SOLIDWORKS_DRAWING("slddrw", MediaType.application("sldworks"));
+ SOLIDWORKS_DRAWING("slddrw", MediaType.application("sldworks")),
+ GRAPH("", MediaType.application("vnd.ms-graph"));
private final String extension;
private final MediaType type;
http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/POIFSContainerDetector.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/POIFSContainerDetector.java b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/POIFSContainerDetector.java
index 0f8a66b..6770cd1 100644
--- a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/POIFSContainerDetector.java
+++ b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/POIFSContainerDetector.java
@@ -69,6 +69,10 @@ public class POIFSContainerDetector implements Detector {
public static final MediaType COMP_OBJ =
new MediaType(GENERAL_EMBEDDED, "format", "comp_obj");
/**
+ * Graph/Charts embedded in PowerPoint and Excel
+ */
+ public static final MediaType MS_GRAPH_CHART = application("vnd.ms-graph");
+ /**
* Microsoft Excel
*/
public static final MediaType XLS = application("vnd.ms-excel");
@@ -150,6 +154,15 @@ public class POIFSContainerDetector implements Detector {
private static final byte[] WORKS_QUILL96 = new byte[]{
0x51, 0x75, 0x69, 0x6c, 0x6c, 0x39, 0x36
};
+
+ /**
+ * An ASCII String "MSGraph.Chart" for embedded MSGraph files
+ * The full designator includes a version, e.g. MSGraph.Chart.8
+ */
+ private static final byte[] MS_GRAPH_CHART_BYTES = new byte[]{
+ 0x4D, 0x53, 0x47, 0x72, 0x61, 0x70, 0x68,
+ 0x2E, 0x43, 0x68, 0x61, 0x72, 0x74,
+ };
/**
* Regexp for matching the MPP Project Data stream
*/
@@ -209,6 +222,10 @@ public class POIFSContainerDetector implements Detector {
// we want to avoid classifying this as Excel
return XLR;
} else if (names.contains("Workbook") || names.contains("WORKBOOK")) {
+ MediaType tmp = processCompObjFormatType(root);
+ if (tmp.equals(MS_GRAPH_CHART)) {
+ return MS_GRAPH_CHART;
+ }
return XLS;
} else if (names.contains("Book")) {
// Excel 95 or older, we won't be able to parse this....
@@ -309,7 +326,9 @@ public class POIFSContainerDetector implements Detector {
* application used to create this file. We want to search for that
* name.
*/
- if (arrayContains(bytes, STAR_DRAW)) {
+ if (arrayContains(bytes, MS_GRAPH_CHART_BYTES)) {
+ return MS_GRAPH_CHART;
+ } else if (arrayContains(bytes, STAR_DRAW)) {
return SDA;
} else if (arrayContains(bytes, STAR_IMPRESS)) {
return SDD;
http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/AbstractPOIContainerExtractionTest.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/AbstractPOIContainerExtractionTest.java b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/AbstractPOIContainerExtractionTest.java
index 51fae83..2b3d141 100644
--- a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/AbstractPOIContainerExtractionTest.java
+++ b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/AbstractPOIContainerExtractionTest.java
@@ -21,7 +21,7 @@ import static org.junit.Assert.assertNotNull;
import java.net.URL;
-import org.apache.tika.TikaTest.TrackingHandler;
+import org.apache.tika.TikaTest;
import org.apache.tika.extractor.ContainerExtractor;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.mime.MediaType;
@@ -30,7 +30,7 @@ import org.apache.tika.mime.MediaType;
* Parent class of tests that the various POI powered parsers are
* able to extract their embedded contents.
*/
-public abstract class AbstractPOIContainerExtractionTest {
+public abstract class AbstractPOIContainerExtractionTest extends TikaTest {
public static final MediaType TYPE_DOC = MediaType.application("msword");
public static final MediaType TYPE_PPT = MediaType.application("vnd.ms-powerpoint");
public static final MediaType TYPE_XLS = MediaType.application("vnd.ms-excel");
http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java
index 2db9ba2..3d28b35 100644
--- a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java
+++ b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java
@@ -20,22 +20,15 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import java.io.InputStream;
import java.util.List;
-import org.apache.tika.TikaTest.TrackingHandler;
import org.apache.tika.extractor.ContainerExtractor;
import org.apache.tika.extractor.ParserContainerExtractor;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaMetadataKeys;
import org.apache.tika.mime.MediaType;
-import org.apache.tika.parser.AutoDetectParser;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
import org.apache.tika.parser.RecursiveParserWrapper;
-import org.apache.tika.sax.BasicContentHandlerFactory;
import org.junit.Test;
-import org.xml.sax.helpers.DefaultHandler;
/**
* Tests that the various POI powered parsers are
@@ -357,16 +350,7 @@ public class POIContainerExtractionTest extends AbstractPOIContainerExtractionTe
@Test
public void testEmbeddedStorageId() throws Exception {
- Parser p = new AutoDetectParser();
-
- RecursiveParserWrapper w = new RecursiveParserWrapper(p,
- new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.XML, -1));
- try (InputStream is = getTestFile("testWORD_embeded.doc")) {
- Metadata meta = new Metadata();
- ParseContext c = new ParseContext();
- w.parse(is, new DefaultHandler(), meta, c);
- }
- List<Metadata> list = w.getMetadata();
+ List<Metadata> list = getRecursiveJson("testWORD_embeded.doc");
//.docx
assertEquals("{F4754C9B-64F5-4B40-8AF4-679732AC0607}",
list.get(10).get(TikaMetadataKeys.EMBEDDED_STORAGE_CLASS_ID));
@@ -378,4 +362,21 @@ public class POIContainerExtractionTest extends AbstractPOIContainerExtractionTe
list.get(16).get(TikaMetadataKeys.EMBEDDED_STORAGE_CLASS_ID));
}
+
+ @Test
+ public void testEmbeddedGraphChart() throws Exception {
+ //doc converts a chart to a actual xls file
+ //so we only need to look in ppt and xls
+ for (String suffix : new String[]{"ppt", "xls"}) {
+ List<Metadata> list = getRecursiveJson("testMSChart-govdocs-428996."+suffix);
+ boolean found = false;
+ for (Metadata m : list) {
+ if (m.get(Metadata.CONTENT_TYPE).equals(POIFSContainerDetector.MS_GRAPH_CHART.toString())) {
+ found = true;
+ }
+ assertNull(m.get(RecursiveParserWrapper.EMBEDDED_EXCEPTION));
+ }
+ assertTrue("didn't find chart in "+suffix, found);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLContainerExtractionTest.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLContainerExtractionTest.java b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLContainerExtractionTest.java
index b90bd54..84465ae 100644
--- a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLContainerExtractionTest.java
+++ b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLContainerExtractionTest.java
@@ -18,12 +18,17 @@ package org.apache.tika.parser.microsoft.ooxml;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
import org.apache.tika.Tika;
-import org.apache.tika.TikaTest.TrackingHandler;
import org.apache.tika.extractor.ContainerExtractor;
import org.apache.tika.extractor.ParserContainerExtractor;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.RecursiveParserWrapper;
import org.apache.tika.parser.microsoft.AbstractPOIContainerExtractionTest;
+import org.apache.tika.parser.microsoft.POIFSContainerDetector;
import org.junit.Before;
import org.junit.Test;
@@ -298,4 +303,20 @@ public class OOXMLContainerExtractionTest extends AbstractPOIContainerExtraction
assertEquals(TYPE_PDF, handler.mediaTypes.get(1));
}
+ @Test
+ public void testEmbeddedGraphChart() throws Exception {
+ //docx converts a chart to a actual xlsx file
+ //so we only need to look in pptx and xlsx
+ for (String suffix : new String[]{"pptx", "xlsx"}) {
+ List<Metadata> list = getRecursiveJson("testMSChart-govdocs-428996."+suffix);
+ boolean found = false;
+ for (Metadata m : list) {
+ if (m.get(Metadata.CONTENT_TYPE).equals(POIFSContainerDetector.MS_GRAPH_CHART.toString())) {
+ found = true;
+ }
+ assertNull(m.get(RecursiveParserWrapper.EMBEDDED_EXCEPTION));
+ }
+ assertTrue("didn't find chart in "+suffix, found);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.ppt
----------------------------------------------------------------------
diff --git a/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.ppt b/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.ppt
new file mode 100644
index 0000000..2b5a3c3
Binary files /dev/null and b/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.ppt differ
http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.pptx
----------------------------------------------------------------------
diff --git a/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.pptx b/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.pptx
new file mode 100644
index 0000000..c1309ba
Binary files /dev/null and b/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.pptx differ
http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.xls
----------------------------------------------------------------------
diff --git a/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.xls b/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.xls
new file mode 100644
index 0000000..7984de8
Binary files /dev/null and b/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.xls differ
http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.xlsx
----------------------------------------------------------------------
diff --git a/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.xlsx b/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.xlsx
new file mode 100644
index 0000000..1882bad
Binary files /dev/null and b/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.xlsx differ