You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2016/04/05 17:15:52 UTC

tika git commit: TIKA-1033 -- add identification for embedded MSChart.Graph files.

Repository: tika
Updated Branches:
  refs/heads/2.x 33b9e483d -> 862234289


TIKA-1033 -- add identification for embedded MSChart.Graph files.


Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/86223428
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/86223428
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/86223428

Branch: refs/heads/2.x
Commit: 862234289514dede8362c04f64305a47b0580ec8
Parents: 33b9e48
Author: tballison <ta...@mitre.org>
Authored: Tue Apr 5 11:15:42 2016 -0400
Committer: tballison <ta...@mitre.org>
Committed: Tue Apr 5 11:15:42 2016 -0400

----------------------------------------------------------------------
 CHANGES.txt                                     |   2 ++
 .../src/test/java/org/apache/tika/TikaTest.java |  25 +++++++++++++
 .../tika/parser/microsoft/OfficeParser.java     |   3 +-
 .../microsoft/POIFSContainerDetector.java       |  21 ++++++++++-
 .../AbstractPOIContainerExtractionTest.java     |   4 +--
 .../microsoft/POIContainerExtractionTest.java   |  35 ++++++++++---------
 .../ooxml/OOXMLContainerExtractionTest.java     |  23 +++++++++++-
 .../testMSChart-govdocs-428996.ppt              | Bin 0 -> 41472 bytes
 .../testMSChart-govdocs-428996.pptx             | Bin 0 -> 56224 bytes
 .../testMSChart-govdocs-428996.xls              | Bin 0 -> 35328 bytes
 .../testMSChart-govdocs-428996.xlsx             | Bin 0 -> 17112 bytes
 11 files changed, 91 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index bc9db91..8459de1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -10,6 +10,8 @@ Release 2.0 - Future Development
 
 Release 1.13 - ???
 
+  * Add detection for embedded MSChart.Graph files (TIKA-1033).
+
   * Fix NPE in Sqlite parser from Nick C (TIKA-1927).
 
   * Fix NPE in Open Document parser from Nick C (TIKA-1916).

http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-core/src/test/java/org/apache/tika/TikaTest.java
----------------------------------------------------------------------
diff --git a/tika-core/src/test/java/org/apache/tika/TikaTest.java b/tika-core/src/test/java/org/apache/tika/TikaTest.java
index 77215d4..fecfdf6 100644
--- a/tika-core/src/test/java/org/apache/tika/TikaTest.java
+++ b/tika-core/src/test/java/org/apache/tika/TikaTest.java
@@ -43,9 +43,12 @@ import org.apache.tika.mime.MediaType;
 import org.apache.tika.parser.AutoDetectParser;
 import org.apache.tika.parser.ParseContext;
 import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.RecursiveParserWrapper;
+import org.apache.tika.sax.BasicContentHandlerFactory;
 import org.apache.tika.sax.BodyContentHandler;
 import org.apache.tika.sax.ToXMLContentHandler;
 import org.xml.sax.ContentHandler;
+import org.xml.sax.helpers.DefaultHandler;
 
 /**
  * Parent class of Tika tests
@@ -194,6 +197,16 @@ public abstract class TikaTest {
         return getText(is, parser, new ParseContext(), new Metadata());
     }
 
+    protected List<Metadata> getRecursiveJson(String filePath) throws Exception {
+        Parser p = new AutoDetectParser();
+        RecursiveParserWrapper wrapper = new RecursiveParserWrapper(p,
+                new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.XML, -1));
+        try (InputStream is = getResourceAsStream("/test-documents/" + filePath)) {
+            wrapper.parse(is, new DefaultHandler(), new Metadata(), new ParseContext());
+        }
+        return wrapper.getMetadata();
+    }
+
     /**
      * Keeps track of media types and file names recursively.
      *
@@ -247,4 +260,16 @@ public abstract class TikaTest {
             }
         }
     }
+
+    public static void debug(List<Metadata> list) {
+        int i = 0;
+        for (Metadata m : list) {
+            for (String n : m.names()) {
+                for (String v : m.getValues(n)) {
+                    System.out.println(i + ": "+n + " : "+v);
+                }
+            }
+            i++;
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java
index 9dd0125..84ce39e 100644
--- a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java
+++ b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java
@@ -229,7 +229,8 @@ public class OfficeParser extends AbstractParser {
         OUTLOOK("msg", MediaType.application("vnd.ms-outlook")),
         SOLIDWORKS_PART("sldprt", MediaType.application("sldworks")),
         SOLIDWORKS_ASSEMBLY("sldasm", MediaType.application("sldworks")),
-        SOLIDWORKS_DRAWING("slddrw", MediaType.application("sldworks"));
+        SOLIDWORKS_DRAWING("slddrw", MediaType.application("sldworks")),
+        GRAPH("", MediaType.application("vnd.ms-graph"));
 
         private final String extension;
         private final MediaType type;

http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/POIFSContainerDetector.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/POIFSContainerDetector.java b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/POIFSContainerDetector.java
index 0f8a66b..6770cd1 100644
--- a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/POIFSContainerDetector.java
+++ b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/POIFSContainerDetector.java
@@ -69,6 +69,10 @@ public class POIFSContainerDetector implements Detector {
     public static final MediaType COMP_OBJ =
             new MediaType(GENERAL_EMBEDDED, "format", "comp_obj");
     /**
+     * Graph/Charts embedded in PowerPoint and Excel
+     */
+    public static final MediaType MS_GRAPH_CHART = application("vnd.ms-graph");
+    /**
      * Microsoft Excel
      */
     public static final MediaType XLS = application("vnd.ms-excel");
@@ -150,6 +154,15 @@ public class POIFSContainerDetector implements Detector {
     private static final byte[] WORKS_QUILL96 = new byte[]{
             0x51, 0x75, 0x69, 0x6c, 0x6c, 0x39, 0x36
     };
+
+    /**
+     * An ASCII String "MSGraph.Chart" for embedded MSGraph files
+     * The full designator includes a version, e.g. MSGraph.Chart.8
+     */
+    private static final byte[] MS_GRAPH_CHART_BYTES = new byte[]{
+            0x4D, 0x53, 0x47, 0x72, 0x61, 0x70, 0x68,
+            0x2E, 0x43, 0x68, 0x61, 0x72, 0x74,
+    };
     /**
      * Regexp for matching the MPP Project Data stream
      */
@@ -209,6 +222,10 @@ public class POIFSContainerDetector implements Detector {
                 // we want to avoid classifying this as Excel
                 return XLR;
             } else if (names.contains("Workbook") || names.contains("WORKBOOK")) {
+                MediaType tmp = processCompObjFormatType(root);
+                if (tmp.equals(MS_GRAPH_CHART)) {
+                    return MS_GRAPH_CHART;
+                }
                 return XLS;
             } else if (names.contains("Book")) {
                 // Excel 95 or older, we won't be able to parse this....
@@ -309,7 +326,9 @@ public class POIFSContainerDetector implements Detector {
                  * application used to create this file. We want to search for that
                  * name.
                  */
-                if (arrayContains(bytes, STAR_DRAW)) {
+                if (arrayContains(bytes, MS_GRAPH_CHART_BYTES)) {
+                    return MS_GRAPH_CHART;
+                } else if (arrayContains(bytes, STAR_DRAW)) {
                     return SDA;
                 } else if (arrayContains(bytes, STAR_IMPRESS)) {
                     return SDD;

http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/AbstractPOIContainerExtractionTest.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/AbstractPOIContainerExtractionTest.java b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/AbstractPOIContainerExtractionTest.java
index 51fae83..2b3d141 100644
--- a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/AbstractPOIContainerExtractionTest.java
+++ b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/AbstractPOIContainerExtractionTest.java
@@ -21,7 +21,7 @@ import static org.junit.Assert.assertNotNull;
 
 import java.net.URL;
 
-import org.apache.tika.TikaTest.TrackingHandler;
+import org.apache.tika.TikaTest;
 import org.apache.tika.extractor.ContainerExtractor;
 import org.apache.tika.io.TikaInputStream;
 import org.apache.tika.mime.MediaType;
@@ -30,7 +30,7 @@ import org.apache.tika.mime.MediaType;
  * Parent class of tests that the various POI powered parsers are
  * able to extract their embedded contents.
  */
-public abstract class AbstractPOIContainerExtractionTest {
+public abstract class AbstractPOIContainerExtractionTest extends TikaTest {
     public static final MediaType TYPE_DOC = MediaType.application("msword");
     public static final MediaType TYPE_PPT = MediaType.application("vnd.ms-powerpoint");
     public static final MediaType TYPE_XLS = MediaType.application("vnd.ms-excel");

http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java
index 2db9ba2..3d28b35 100644
--- a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java
+++ b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java
@@ -20,22 +20,15 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
-import java.io.InputStream;
 import java.util.List;
 
-import org.apache.tika.TikaTest.TrackingHandler;
 import org.apache.tika.extractor.ContainerExtractor;
 import org.apache.tika.extractor.ParserContainerExtractor;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.metadata.TikaMetadataKeys;
 import org.apache.tika.mime.MediaType;
-import org.apache.tika.parser.AutoDetectParser;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
 import org.apache.tika.parser.RecursiveParserWrapper;
-import org.apache.tika.sax.BasicContentHandlerFactory;
 import org.junit.Test;
-import org.xml.sax.helpers.DefaultHandler;
 
 /**
  * Tests that the various POI powered parsers are
@@ -357,16 +350,7 @@ public class POIContainerExtractionTest extends AbstractPOIContainerExtractionTe
     @Test
     public void testEmbeddedStorageId() throws Exception {
 
-        Parser p = new AutoDetectParser();
-
-        RecursiveParserWrapper w = new RecursiveParserWrapper(p,
-                new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.XML, -1));
-        try (InputStream is = getTestFile("testWORD_embeded.doc")) {
-            Metadata meta = new Metadata();
-            ParseContext c = new ParseContext();
-            w.parse(is, new DefaultHandler(), meta, c);
-        }
-        List<Metadata> list = w.getMetadata();
+        List<Metadata> list = getRecursiveJson("testWORD_embeded.doc");
         //.docx
         assertEquals("{F4754C9B-64F5-4B40-8AF4-679732AC0607}",
                 list.get(10).get(TikaMetadataKeys.EMBEDDED_STORAGE_CLASS_ID));
@@ -378,4 +362,21 @@ public class POIContainerExtractionTest extends AbstractPOIContainerExtractionTe
                 list.get(16).get(TikaMetadataKeys.EMBEDDED_STORAGE_CLASS_ID));
 
     }
+
+    @Test
+    public void testEmbeddedGraphChart() throws Exception {
+        //doc converts a chart to a actual xls file
+        //so we only need to look in ppt and xls
+        for (String suffix : new String[]{"ppt", "xls"}) {
+            List<Metadata> list = getRecursiveJson("testMSChart-govdocs-428996."+suffix);
+            boolean found = false;
+            for (Metadata m : list) {
+                if (m.get(Metadata.CONTENT_TYPE).equals(POIFSContainerDetector.MS_GRAPH_CHART.toString())) {
+                    found = true;
+                }
+                assertNull(m.get(RecursiveParserWrapper.EMBEDDED_EXCEPTION));
+            }
+            assertTrue("didn't find chart in "+suffix, found);
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLContainerExtractionTest.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLContainerExtractionTest.java b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLContainerExtractionTest.java
index b90bd54..84465ae 100644
--- a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLContainerExtractionTest.java
+++ b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLContainerExtractionTest.java
@@ -18,12 +18,17 @@ package org.apache.tika.parser.microsoft.ooxml;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
 
 import org.apache.tika.Tika;
-import org.apache.tika.TikaTest.TrackingHandler;
 import org.apache.tika.extractor.ContainerExtractor;
 import org.apache.tika.extractor.ParserContainerExtractor;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.RecursiveParserWrapper;
 import org.apache.tika.parser.microsoft.AbstractPOIContainerExtractionTest;
+import org.apache.tika.parser.microsoft.POIFSContainerDetector;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -298,4 +303,20 @@ public class OOXMLContainerExtractionTest extends AbstractPOIContainerExtraction
         assertEquals(TYPE_PDF, handler.mediaTypes.get(1));
     }
 
+    @Test
+    public void testEmbeddedGraphChart() throws Exception {
+        //docx converts a chart to a actual xlsx file
+        //so we only need to look in pptx and xlsx
+        for (String suffix : new String[]{"pptx", "xlsx"}) {
+            List<Metadata> list = getRecursiveJson("testMSChart-govdocs-428996."+suffix);
+            boolean found = false;
+            for (Metadata m : list) {
+                if (m.get(Metadata.CONTENT_TYPE).equals(POIFSContainerDetector.MS_GRAPH_CHART.toString())) {
+                    found = true;
+                }
+                assertNull(m.get(RecursiveParserWrapper.EMBEDDED_EXCEPTION));
+            }
+            assertTrue("didn't find chart in "+suffix, found);
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.ppt
----------------------------------------------------------------------
diff --git a/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.ppt b/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.ppt
new file mode 100644
index 0000000..2b5a3c3
Binary files /dev/null and b/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.ppt differ

http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.pptx
----------------------------------------------------------------------
diff --git a/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.pptx b/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.pptx
new file mode 100644
index 0000000..c1309ba
Binary files /dev/null and b/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.pptx differ

http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.xls
----------------------------------------------------------------------
diff --git a/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.xls b/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.xls
new file mode 100644
index 0000000..7984de8
Binary files /dev/null and b/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.xls differ

http://git-wip-us.apache.org/repos/asf/tika/blob/86223428/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.xlsx
----------------------------------------------------------------------
diff --git a/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.xlsx b/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.xlsx
new file mode 100644
index 0000000..1882bad
Binary files /dev/null and b/tika-test-resources/src/test/resources/test-documents/testMSChart-govdocs-428996.xlsx differ