You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ni...@apache.org on 2017/10/18 14:51:54 UTC
[tika] branch master updated (ad23d84 -> 0d92bc8)
This is an automated email from the ASF dual-hosted git repository.
nick pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git.
from ad23d84 TIKA-2469 -- narrow mime detection for ms-owner files and add detection for nls files.
new 17e4b66 A dummy parser unit test for iWorks 13
new 5c7547b Have the iWorks 13 parser set the content type on the metadata if possible, otherwise remains no-op
new 0d92bc8 Add notes on why we can't get the Numbers or Pages type just yet - need to call out to another library or decode the Document.iwa snappy stream ourselves
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../parser/iwork/iwana/IWork13PackageParser.java | 105 +++++++++++++++++----
.../tika/parser/pkg/ZipContainerDetector.java | 1 -
.../tika/detect/TestContainerAwareDetector.java | 2 +
.../tika/parser/iwork/iwana/IWork13ParserTest.java | 88 +++++++++++++++++
4 files changed, 179 insertions(+), 17 deletions(-)
create mode 100644 tika-parsers/src/test/java/org/apache/tika/parser/iwork/iwana/IWork13ParserTest.java
--
To stop receiving notification emails like this one, please contact
['"commits@tika.apache.org" <co...@tika.apache.org>'].
[tika] 02/03: Have the iWorks 13 parser set the content type on the
metadata if possible, otherwise remains no-op
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nick pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 5c7547bac9208082920859a5040a8b9fa31da642
Author: Nick Burch <ni...@apache.org>
AuthorDate: Wed Oct 18 14:59:35 2017 +0100
Have the iWorks 13 parser set the content type on the metadata if possible, otherwise remains no-op
---
.../parser/iwork/iwana/IWork13PackageParser.java | 81 +++++++++++++++++++---
.../tika/parser/pkg/ZipContainerDetector.java | 1 -
.../tika/parser/iwork/iwana/IWork13ParserTest.java | 6 +-
3 files changed, 77 insertions(+), 11 deletions(-)
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/iwork/iwana/IWork13PackageParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/iwork/iwana/IWork13PackageParser.java
index 637b51b..b96cc39 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/iwork/iwana/IWork13PackageParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/iwork/iwana/IWork13PackageParser.java
@@ -20,6 +20,7 @@ package org.apache.tika.parser.iwork.iwana;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipFile;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.AbstractParser;
@@ -31,8 +32,11 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Enumeration;
import java.util.HashSet;
import java.util.Set;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
public class IWork13PackageParser extends AbstractParser {
@@ -53,13 +57,35 @@ public class IWork13PackageParser extends AbstractParser {
}
public static MediaType detect(ZipFile zipFile) {
- ZipArchiveEntry entry = zipFile.getEntry("Index/MasterSlide.iwa");
- if (zipFile.getEntry("Index/MasterSlide.iwa") != null ||
- zipFile.getEntry("Index/Slide.iwa") != null) {
- return KEYNOTE13.getType();
- }
- //TODO: figure out how to distinguish numbers from pages
- return UNKNOWN13.getType();
+ MediaType type = null;
+ Enumeration<? extends ZipEntry> entries = zipFile.getEntries();
+ while (entries.hasMoreElements()) {
+ ZipEntry entry = entries.nextElement();
+ type = IWork13DocumentType.detectIfPossible(entry);
+ if (type != null) return type;
+ }
+ return UNKNOWN13.getType();
+ }
+
+ /**
+ * @return Specific type if this identifies one, otherwise null
+ */
+ public static MediaType detectIfPossible(ZipEntry entry) {
+ String name = entry.getName();
+ if (! name.endsWith(".iwa")) return null;
+
+ if (name.equals("Index/MasterSlide.iwa") ||
+ name.startsWith("Index/MasterSlide-")) {
+ return KEYNOTE13.getType();
+ }
+ if (name.equals("Index/Slide.iwa") ||
+ name.startsWith("Index/Slide-")) {
+ return KEYNOTE13.getType();
+ }
+ //TODO: figure out how to distinguish numbers from pages
+
+ // Unknown
+ return null;
}
}
@@ -81,6 +107,45 @@ public class IWork13PackageParser extends AbstractParser {
@Override
public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException {
- //no-op for now
+ // Open the Zip stream
+ // Use a File if we can, and an already open zip is even better
+ ZipFile zipFile = null;
+ ZipInputStream zipStream = null;
+ if (stream instanceof TikaInputStream) {
+ TikaInputStream tis = (TikaInputStream) stream;
+ Object container = ((TikaInputStream) stream).getOpenContainer();
+ if (container instanceof ZipFile) {
+ zipFile = (ZipFile) container;
+ } else if (tis.hasFile()) {
+ zipFile = new ZipFile(tis.getFile());
+ } else {
+ zipStream = new ZipInputStream(stream);
+ }
+ } else {
+ zipStream = new ZipInputStream(stream);
+ }
+
+ // For now, just detect
+ MediaType type = null;
+ if (zipFile != null) {
+ Enumeration<? extends ZipEntry> entries = zipFile.getEntries();
+ while (entries.hasMoreElements()) {
+ ZipEntry entry = entries.nextElement();
+ if (type == null) {
+ type = IWork13DocumentType.detectIfPossible(entry);
+ }
+ }
+ } else {
+ ZipEntry entry = zipStream.getNextEntry();
+ while (entry != null) {
+ if (type == null) {
+ type = IWork13DocumentType.detectIfPossible(entry);
+ }
+ entry = zipStream.getNextEntry();
+ }
+ }
+ if (type != null) {
+ metadata.add(Metadata.CONTENT_TYPE, type.toString());
+ }
}
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ZipContainerDetector.java b/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ZipContainerDetector.java
index 3f9211b..9a5befa 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ZipContainerDetector.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ZipContainerDetector.java
@@ -292,7 +292,6 @@ public class ZipContainerDetector implements Detector {
return IWork13PackageParser.IWork13DocumentType.detect(zip);
}
return null;
-
}
private static MediaType detectIWork(ZipFile zip) {
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/iwork/iwana/IWork13ParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/iwork/iwana/IWork13ParserTest.java
index c671253..4bbbcbf 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/iwork/iwana/IWork13ParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/iwork/iwana/IWork13ParserTest.java
@@ -57,9 +57,11 @@ public class IWork13ParserTest {
iWorkParser.parse(input, handler, metadata, parseContext);
// Currently parsing is a no-op
- // TODO Test properly when a full Parser is added
- assertEquals(0, metadata.size());
+ // Will only get type
+ assertEquals(1, metadata.size());
assertEquals("", handler.toString());
+ assertEquals(IWork13PackageParser.IWork13DocumentType.KEYNOTE13.getType().toString(),
+ metadata.get(Metadata.CONTENT_TYPE));
}
@Test
--
To stop receiving notification emails like this one, please contact
"commits@tika.apache.org" <co...@tika.apache.org>.
[tika] 01/03: A dummy parser unit test for iWorks 13
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nick pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 17e4b66410a3bca6c749dda8c49bdb41f3d1b609
Author: Nick Burch <ni...@apache.org>
AuthorDate: Wed Oct 18 14:31:34 2017 +0100
A dummy parser unit test for iWorks 13
---
.../tika/parser/iwork/iwana/IWork13ParserTest.java | 90 ++++++++++++++++++++++
1 file changed, 90 insertions(+)
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/iwork/iwana/IWork13ParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/iwork/iwana/IWork13ParserTest.java
new file mode 100644
index 0000000..c671253
--- /dev/null
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/iwork/iwana/IWork13ParserTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.iwork.iwana;
+
+import static org.apache.tika.TikaTest.assertContains;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.sax.BodyContentHandler;
+import org.junit.Before;
+import org.junit.Test;
+import org.xml.sax.ContentHandler;
+
+/**
+ * Limited testing for the iWorks 13 format parser, which
+ * currently doesn't do anything more than detection....
+ */
+public class IWork13ParserTest {
+ private IWork13PackageParser iWorkParser;
+ private ParseContext parseContext;
+
+ @Before
+ public void setUp() {
+ iWorkParser = new IWork13PackageParser();
+ parseContext = new ParseContext();
+ parseContext.set(Parser.class, new AutoDetectParser());
+ }
+
+ @Test
+ public void testParseKeynote13() throws Exception {
+ InputStream input = IWork13ParserTest.class.getResourceAsStream("/test-documents/testKeynote2013.key");
+ Metadata metadata = new Metadata();
+ ContentHandler handler = new BodyContentHandler();
+ iWorkParser.parse(input, handler, metadata, parseContext);
+
+ // Currently parsing is a no-op
+ // TODO Test properly when a full Parser is added
+ assertEquals(0, metadata.size());
+ assertEquals("", handler.toString());
+ }
+
+ @Test
+ public void testParseNumbers13() throws Exception {
+ InputStream input = IWork13ParserTest.class.getResourceAsStream("/test-documents/testNumbers2013.numbers");
+ Metadata metadata = new Metadata();
+ ContentHandler handler = new BodyContentHandler();
+ iWorkParser.parse(input, handler, metadata, parseContext);
+
+ // Currently parsing is a no-op
+ // TODO Test properly when a full Parser is added
+ assertEquals(0, metadata.size());
+ assertEquals("", handler.toString());
+ }
+
+ @Test
+ public void testParsePages13() throws Exception {
+ InputStream input = IWork13ParserTest.class.getResourceAsStream("/test-documents/testPages2013.pages");
+ Metadata metadata = new Metadata();
+ ContentHandler handler = new BodyContentHandler();
+ iWorkParser.parse(input, handler, metadata, parseContext);
+
+ // Currently parsing is a no-op
+ // TODO Test properly when a full Parser is added
+ assertEquals(0, metadata.size());
+ assertEquals("", handler.toString());
+ }
+}
--
To stop receiving notification emails like this one, please contact
"commits@tika.apache.org" <co...@tika.apache.org>.
[tika] 03/03: Add notes on why we can't get the Numbers or Pages
type just yet - need to call out to another library or decode the
Document.iwa snappy stream ourselves
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nick pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 0d92bc862c3c344d65d3f6c260b0f5ea4c389fc0
Author: Nick Burch <ni...@apache.org>
AuthorDate: Wed Oct 18 15:50:59 2017 +0100
Add notes on why we can't get the Numbers or Pages type just yet - need to call out to another library or decode the Document.iwa snappy stream ourselves
---
.../parser/iwork/iwana/IWork13PackageParser.java | 34 +++++++++++++---------
.../tika/detect/TestContainerAwareDetector.java | 2 ++
.../tika/parser/iwork/iwana/IWork13ParserTest.java | 14 ++++-----
3 files changed, 28 insertions(+), 22 deletions(-)
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/iwork/iwana/IWork13PackageParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/iwork/iwana/IWork13PackageParser.java
index b96cc39..a090e84 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/iwork/iwana/IWork13PackageParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/iwork/iwana/IWork13PackageParser.java
@@ -17,17 +17,6 @@
package org.apache.tika.parser.iwork.iwana;
-import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
-import org.apache.commons.compress.archivers.zip.ZipFile;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.mime.MediaType;
-import org.apache.tika.parser.AbstractParser;
-import org.apache.tika.parser.ParseContext;
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
@@ -38,6 +27,16 @@ import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
+import org.apache.commons.compress.archivers.zip.ZipFile;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.AbstractParser;
+import org.apache.tika.parser.ParseContext;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+
public class IWork13PackageParser extends AbstractParser {
public enum IWork13DocumentType {
@@ -64,16 +63,19 @@ public class IWork13PackageParser extends AbstractParser {
type = IWork13DocumentType.detectIfPossible(entry);
if (type != null) return type;
}
+
+ // If we get here, we don't know what it is
return UNKNOWN13.getType();
}
/**
* @return Specific type if this identifies one, otherwise null
*/
- public static MediaType detectIfPossible(ZipEntry entry) {
+ protected static MediaType detectIfPossible(ZipEntry entry) {
String name = entry.getName();
if (! name.endsWith(".iwa")) return null;
+ // Is it a uniquely identifying filename?
if (name.equals("Index/MasterSlide.iwa") ||
name.startsWith("Index/MasterSlide-")) {
return KEYNOTE13.getType();
@@ -82,7 +84,13 @@ public class IWork13PackageParser extends AbstractParser {
name.startsWith("Index/Slide-")) {
return KEYNOTE13.getType();
}
- //TODO: figure out how to distinguish numbers from pages
+
+ // Is it the main document?
+ if (name.equals("Index/Document.iwa")) {
+ // TODO Decode the snappy stream, and check for the Message Type
+ // = 2 (TN::SheetArchive), it is a numbers file;
+ // = 10000 (TP::DocumentArchive), that's a pages file
+ }
// Unknown
return null;
diff --git a/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java b/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java
index b6a79eb..e4117c4 100644
--- a/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java
+++ b/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java
@@ -320,6 +320,8 @@ public class TestContainerAwareDetector {
public void testDetectIWork2013() throws Exception {
assertTypeByData("testKeynote2013.key",
IWork13PackageParser.IWork13DocumentType.KEYNOTE13.getType().toString());
+ // Without decoding the Document snappy stream, we can't tell the
+ // difference between these two just based on the zip entries
assertTypeByData("testNumbers2013.numbers",
IWork13PackageParser.IWork13DocumentType.UNKNOWN13.getType().toString());
assertTypeByData("testPages2013.pages",
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/iwork/iwana/IWork13ParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/iwork/iwana/IWork13ParserTest.java
index 4bbbcbf..60477a5 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/iwork/iwana/IWork13ParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/iwork/iwana/IWork13ParserTest.java
@@ -16,16 +16,11 @@
*/
package org.apache.tika.parser.iwork.iwana;
-import static org.apache.tika.TikaTest.assertContains;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
import java.io.InputStream;
-import java.util.Arrays;
-import java.util.List;
import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
@@ -56,8 +51,7 @@ public class IWork13ParserTest {
ContentHandler handler = new BodyContentHandler();
iWorkParser.parse(input, handler, metadata, parseContext);
- // Currently parsing is a no-op
- // Will only get type
+ // Currently parsing is a no-op, so will only get the Type
assertEquals(1, metadata.size());
assertEquals("", handler.toString());
assertEquals(IWork13PackageParser.IWork13DocumentType.KEYNOTE13.getType().toString(),
@@ -71,7 +65,8 @@ public class IWork13ParserTest {
ContentHandler handler = new BodyContentHandler();
iWorkParser.parse(input, handler, metadata, parseContext);
- // Currently parsing is a no-op
+ // Currently parsing is a no-op, and we can't get the type without
+ // decoding the Snappy stream
// TODO Test properly when a full Parser is added
assertEquals(0, metadata.size());
assertEquals("", handler.toString());
@@ -84,7 +79,8 @@ public class IWork13ParserTest {
ContentHandler handler = new BodyContentHandler();
iWorkParser.parse(input, handler, metadata, parseContext);
- // Currently parsing is a no-op
+ // Currently parsing is a no-op, and we can't get the type without
+ // decoding the Snappy stream
// TODO Test properly when a full Parser is added
assertEquals(0, metadata.size());
assertEquals("", handler.toString());
--
To stop receiving notification emails like this one, please contact
"commits@tika.apache.org" <co...@tika.apache.org>.