You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2016/04/16 03:02:30 UTC
tika git commit: Add DOM and Stax parsers to ParseContext
Repository: tika
Updated Branches:
refs/heads/master e032ac619 -> 3d491317c
Add DOM and Stax parsers to ParseContext
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/3d491317
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/3d491317
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/3d491317
Branch: refs/heads/master
Commit: 3d491317c8c67ce9f159b531ea476e547e0dff83
Parents: e032ac6
Author: tballison <ta...@mitre.org>
Authored: Fri Apr 15 21:02:06 2016 -0400
Committer: tballison <ta...@mitre.org>
Committed: Fri Apr 15 21:02:06 2016 -0400
----------------------------------------------------------------------
.../org/apache/tika/parser/ParseContext.java | 134 ++++++++++++++++++-
.../org/apache/tika/parser/pdf/PDF2XHTML.java | 7 +-
.../org/apache/tika/parser/pdf/PDFParser.java | 54 +++++---
.../apache/tika/parser/pdf/XFAExtractor.java | 30 ++---
4 files changed, 178 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/3d491317/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
----------------------------------------------------------------------
diff --git a/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java b/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
index 48a7841..2ff07e5 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
@@ -16,16 +16,25 @@
*/
package org.apache.tika.parser;
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-
import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLResolver;
+import javax.xml.stream.XMLStreamException;
+import java.io.IOException;
+import java.io.Serializable;
+import java.io.StringReader;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.tika.exception.TikaException;
+import org.xml.sax.EntityResolver;
+import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.SAXNotRecognizedException;
import org.xml.sax.SAXNotSupportedException;
@@ -43,7 +52,13 @@ public class ParseContext implements Serializable {
/** Map of objects in this context */
private final Map<String, Object> context = new HashMap<String, Object>();
-
+
+ public static final EntityResolver IGNORING_ENTITY_RESOLVER = new EntityResolver() {
+ public InputSource resolveEntity(String publicId, String systemId) throws SAXException, IOException {
+ return new InputSource(new StringReader(""));
+ }
+ };
+
/**
* Adds the given value to the context as an implementation of the given
* interface.
@@ -145,4 +160,113 @@ public class ParseContext implements Serializable {
return factory;
}
+ /**
+ * Returns the DOM builder factory specified in this parsing context.
+ * If a factory is not explicitly specified, then a default factory
+ * instance is created and returned. The default factory instance is
+ * configured to be namespace-aware and to apply reasonable security
+ * features.
+ *
+ * @since Apache Tika 1.13
+ * @return DOM parser factory
+ */
+ public DocumentBuilderFactory getDocumentBuilderFactory() {
+ //borrowed from Apache POI
+ DocumentBuilderFactory documentBuilderFactory = get(DocumentBuilderFactory.class);
+ if (documentBuilderFactory == null) {
+ documentBuilderFactory = DocumentBuilderFactory.newInstance();
+ }
+ documentBuilderFactory.setNamespaceAware(true);
+ documentBuilderFactory.setValidating(false);
+ trySetSAXFeature(documentBuilderFactory, XMLConstants.FEATURE_SECURE_PROCESSING, true);
+ trySetXercesSecurityManager(documentBuilderFactory);
+ return documentBuilderFactory;
+ }
+
+ /**
+ * Returns the DOM builder specified in this parsing context.
+ * If a factory is not explicitly specified, then a builder
+ * instance is created and returned. The builder instance is
+ * configured to apply an {@link #IGNORING_ENTITY_RESOLVER}.
+ *
+ * @since Apache Tika 1.13
+ * @return DOM Builder
+ */
+ public DocumentBuilder getDocumentBuilder() {
+ try {
+ DocumentBuilderFactory documentBuilderFactory = getDocumentBuilderFactory();
+ DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
+ documentBuilder.setEntityResolver(IGNORING_ENTITY_RESOLVER);
+ documentBuilder.setErrorHandler(null);
+ return documentBuilder;
+ } catch (ParserConfigurationException e) {
+ throw new IllegalStateException("cannot create a DocumentBuilder", e);
+ }
+ }
+
+ private static void trySetSAXFeature(DocumentBuilderFactory dbf, String feature, boolean enabled) {
+ try {
+ dbf.setFeature(feature, enabled);
+ } catch (Exception|AbstractMethodError e) {
+ }
+ }
+
+ private static void trySetXercesSecurityManager(DocumentBuilderFactory dbf) {
+ // Try built-in JVM one first, standalone if not
+ for (String securityManagerClassName : new String[] {
+ "com.sun.org.apache.xerces.internal.util.SecurityManager",
+ "org.apache.xerces.util.SecurityManager"
+ }) {
+ try {
+ Object mgr = Class.forName(securityManagerClassName).newInstance();
+ Method setLimit = mgr.getClass().getMethod("setEntityExpansionLimit", Integer.TYPE);
+ setLimit.invoke(mgr, 4096);
+ dbf.setAttribute("http://apache.org/xml/properties/security-manager", mgr);
+ // Stop once one can be setup without error
+ return;
+ } catch (Throwable t) {
+ }
+ }
+ }
+
+ /**
+ * Returns the StAX input factory specified in this parsing context.
+ * If a factory is not explicitly specified, then a default factory
+ * instance is created and returned. The default factory instance is
+ * configured to be namespace-aware and to apply reasonable security
+ * features -- don't support dtd, ignore external entities, null XMLResolver.
+ *
+ * @since Apache Tika 1.13
+ * @return StAX input factory
+ */
+ public XMLInputFactory getXMLInputFactory() {
+ XMLInputFactory factory = get(XMLInputFactory.class);
+ if (factory == null) {
+ factory = XMLInputFactory.newFactory().newFactory();
+ }
+
+ tryToSetProperty(factory, XMLInputFactory.IS_NAMESPACE_AWARE, true);
+ tryToSetProperty(factory, XMLInputFactory.IS_VALIDATING, false);
+
+ tryToSetProperty(factory, XMLInputFactory.SUPPORT_DTD, false);
+ tryToSetProperty(factory, XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
+
+ factory.setXMLResolver(new XMLResolver() {
+ @Override
+ public Object resolveEntity(String publicID, String systemID, String baseURI, String namespace) throws
+ XMLStreamException {
+ return null;
+ }
+ });
+ return factory;
+ }
+
+ private void tryToSetProperty(XMLInputFactory factory, String key, boolean value) {
+ try {
+ factory.setProperty(key, value);
+ } catch (IllegalArgumentException e) {
+ //swallow
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/tika/blob/3d491317/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java b/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
index bd13eeb..fed5137 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
@@ -648,10 +648,11 @@ class PDF2XHTML extends PDFTextStripper {
PDXFAResource pdxfa = form.getXFA();
if (pdxfa != null) {
+ //if successful, return
XFAExtractor xfaExtractor = new XFAExtractor();
- try {
- xfaExtractor.extract(new BufferedInputStream(
- new ByteArrayInputStream(pdxfa.getBytes())), handler, metadata);
+ try (InputStream is = new BufferedInputStream(
+ new ByteArrayInputStream(pdxfa.getBytes()))) {
+ xfaExtractor.extract(is, handler, metadata, context);
return;
} catch (XMLStreamException |IOException e) {
//if there was an xml parse exception in xfa, try the AcroForm
http://git-wip-us.apache.org/repos/asf/tika/blob/3d491317/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
index 17785c9..e102d27 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
@@ -16,6 +16,7 @@
*/
package org.apache.tika.parser.pdf;
+import javax.xml.parsers.DocumentBuilder;
import javax.xml.stream.XMLStreamException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@@ -31,7 +32,6 @@ import org.apache.commons.io.input.CloseShieldInputStream;
import org.apache.jempbox.xmp.XMPMetadata;
import org.apache.jempbox.xmp.XMPSchema;
import org.apache.jempbox.xmp.XMPSchemaDublinCore;
-import org.apache.jempbox.xmp.XMPSchemaMediaManagement;
import org.apache.jempbox.xmp.pdfa.XMPSchemaPDFAId;
import org.apache.pdfbox.cos.COSArray;
import org.apache.pdfbox.cos.COSBase;
@@ -40,6 +40,7 @@ import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.cos.COSString;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDDocumentInformation;
+import org.apache.pdfbox.pdmodel.common.PDMetadata;
import org.apache.pdfbox.pdmodel.encryption.AccessPermission;
import org.apache.pdfbox.pdmodel.encryption.InvalidPasswordException;
import org.apache.tika.exception.EncryptedDocumentException;
@@ -58,7 +59,9 @@ import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.PasswordProvider;
import org.apache.tika.parser.image.xmp.JempboxExtractor;
import org.apache.tika.sax.XHTMLContentHandler;
+import org.w3c.dom.Document;
import org.xml.sax.ContentHandler;
+import org.xml.sax.ErrorHandler;
import org.xml.sax.SAXException;
/**
@@ -98,6 +101,8 @@ public class PDFParser extends AbstractParser {
Collections.singleton(MEDIA_TYPE);
private PDFParserConfig defaultConfig = new PDFParserConfig();
+
+
public Set<MediaType> getSupportedTypes(ParseContext context) {
return SUPPORTED_TYPES;
}
@@ -128,13 +133,13 @@ public class PDFParser extends AbstractParser {
metadata.set("pdf:encrypted", Boolean.toString(pdfDocument.isEncrypted()));
metadata.set(Metadata.CONTENT_TYPE, "application/pdf");
- extractMetadata(pdfDocument, metadata);
+ extractMetadata(pdfDocument, metadata, context);
AccessChecker checker = localConfig.getAccessChecker();
checker.check(metadata);
if (handler != null) {
if (shouldHandleXFAOnly(pdfDocument, localConfig)) {
- handleXFAOnly(pdfDocument, handler, metadata);
+ handleXFAOnly(pdfDocument, handler, metadata, context);
} else {
PDF2XHTML.process(pdfDocument, handler, context, metadata, localConfig);
}
@@ -171,7 +176,7 @@ public class PDFParser extends AbstractParser {
}
- private void extractMetadata(PDDocument document, Metadata metadata)
+ private void extractMetadata(PDDocument document, Metadata metadata, ParseContext context)
throws TikaException {
//first extract AccessPermissions
@@ -195,15 +200,13 @@ public class PDFParser extends AbstractParser {
//now go for the XMP
- org.apache.jempbox.xmp.XMPMetadata xmp = null;
+ Document dom = loadDOM(document.getDocumentCatalog().getMetadata(), context);
+
+ XMPMetadata xmp = null;
+ if (dom != null) {
+ xmp = new XMPMetadata(dom);
+ }
XMPSchemaDublinCore dcSchema = null;
- XMPSchemaMediaManagement mmSchema = null;
- try {
- if (document.getDocumentCatalog().getMetadata() != null) {
- InputStream xmpIs = document.getDocumentCatalog().getMetadata().exportXMPMetadata();
- xmp = XMPMetadata.load(xmpIs);
- }
- } catch (IOException e) {}
if (xmp != null) {
try {
@@ -482,15 +485,15 @@ public class PDFParser extends AbstractParser {
return false;
}
- private void handleXFAOnly(PDDocument pdDocument, ContentHandler handler, Metadata metadata)
+ private void handleXFAOnly(PDDocument pdDocument, ContentHandler handler,
+ Metadata metadata, ParseContext context)
throws SAXException, IOException, TikaException {
XFAExtractor ex = new XFAExtractor();
XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
xhtml.startDocument();
- try {
- ex.extract(new ByteArrayInputStream(
- pdDocument.getDocumentCatalog().getAcroForm().getXFA().getBytes()),
- xhtml, metadata);
+ try (InputStream is = new ByteArrayInputStream(
+ pdDocument.getDocumentCatalog().getAcroForm().getXFA().getBytes())) {
+ ex.extract(is, xhtml, metadata, context);
} catch (XMLStreamException e) {
throw new TikaException("XML error in XFA", e);
}
@@ -589,4 +592,21 @@ public class PDFParser extends AbstractParser {
defaultConfig.setSortByPosition(v);
}
+
+ //can return null!
+ private Document loadDOM(PDMetadata pdMetadata, ParseContext context) {
+ if (pdMetadata == null) {
+ return null;
+ }
+ try (InputStream is = pdMetadata.exportXMPMetadata()) {
+ DocumentBuilder documentBuilder = context.getDocumentBuilder();
+ documentBuilder.setErrorHandler((ErrorHandler)null);
+ return documentBuilder.parse(is);
+ } catch (IOException|SAXException e) {
+ //swallow
+ }
+ return null;
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/tika/blob/3d491317/tika-parsers/src/main/java/org/apache/tika/parser/pdf/XFAExtractor.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/pdf/XFAExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/pdf/XFAExtractor.java
index 3c2b496..d3c34dd 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/pdf/XFAExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/pdf/XFAExtractor.java
@@ -16,19 +16,19 @@
*/
package org.apache.tika.parser.pdf;
+import javax.xml.namespace.QName;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
import java.io.InputStream;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import javax.xml.namespace.QName;
-import javax.xml.stream.XMLInputFactory;
-import javax.xml.stream.XMLResolver;
-import javax.xml.stream.XMLStreamConstants;
-import javax.xml.stream.XMLStreamException;
-import javax.xml.stream.XMLStreamReader;
+
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
import org.apache.tika.sax.XHTMLContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;
@@ -62,20 +62,6 @@ class XFAExtractor {
private static final String FIELD_LN = "field";
private static final QName XFA_DATA = new QName(XFA_DATA_NS, "data");
- private static final XMLInputFactory factory;
-
- static {
- factory = XMLInputFactory.newFactory();
- factory.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, true);
- factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
- factory.setProperty(XMLInputFactory.IS_VALIDATING, false);
- factory.setXMLResolver(new XMLResolver() {
- @Override
- public Object resolveEntity(String publicID, String systemID, String baseURI, String namespace) throws XMLStreamException {
- return null;
- }
- });
- }
private final Matcher xfaTemplateMatcher;//namespace any version
private final Matcher textMatcher;
@@ -84,7 +70,7 @@ class XFAExtractor {
textMatcher = TEXT_PATTERN.matcher("");
}
- void extract(InputStream xfaIs, XHTMLContentHandler xhtml, Metadata m)
+ void extract(InputStream xfaIs, XHTMLContentHandler xhtml, Metadata m, ParseContext context)
throws XMLStreamException, SAXException {
xhtml.startElement("div", "class", "xfa_content");
@@ -99,7 +85,7 @@ class XFAExtractor {
//
//As a final step, dump the merged fields and the values.
- XMLStreamReader reader = factory.createXMLStreamReader(xfaIs);
+ XMLStreamReader reader = context.getXMLInputFactory().createXMLStreamReader(xfaIs);
while (reader.hasNext()) {
switch (reader.next()) {
case XMLStreamConstants.START_ELEMENT :