You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2018/05/21 18:31:50 UTC
[tika] branch branch_1x updated (64fef4e -> c40045a)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a change to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git.
from 64fef4e TIKA-2644 improve api for recursiveparserwrapper -- deconflicted
new c203ef3 TIKA-2645 for tika-core
new 0101164 TIKA-2645 - use a pool for SAXParsers -- tika-parsers package
new aa1a749 TIKA-2645 - use a pool for SAXParsers -- improve comments and avoid permanent hangs if a parser has forgotten to release its SAXParser.
new 017096f TIKA-2645 - remove commons math from dependencies
new c40045a update changes
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.txt | 10 +
.../org/apache/tika/detect/XmlRootExtractor.java | 23 +--
.../java/org/apache/tika/mime/MimeTypesReader.java | 113 +++++++++-
.../java/org/apache/tika/parser/NetworkParser.java | 7 +-
.../java/org/apache/tika/parser/ParseContext.java | 33 +++
.../java/org/apache/tika/utils/XMLReaderUtils.java | 143 +++++++++++--
.../org/apache/tika/MultiThreadedTikaTest.java | 230 +++++++++++++++++----
tika-parsers/pom.xml | 1 -
.../java/org/apache/tika/parser/dif/DIFParser.java | 9 +-
.../apache/tika/parser/epub/EpubContentParser.java | 14 +-
.../tika/parser/iwork/IWorkPackageParser.java | 16 +-
.../microsoft/ooxml/AbstractOOXMLExtractor.java | 8 +-
.../ooxml/SXSLFPowerPointExtractorDecorator.java | 14 +-
.../ooxml/SXWPFWordExtractorDecorator.java | 9 +-
.../ooxml/XSSFExcelExtractorDecorator.java | 12 +-
.../microsoft/ooxml/xps/XPSExtractorDecorator.java | 20 +-
.../microsoft/ooxml/xwpf/XWPFStylesShim.java | 12 +-
.../ooxml/xwpf/ml2006/Word2006MLParser.java | 10 +-
.../microsoft/xml/AbstractXML2003Parser.java | 9 +-
.../apache/tika/parser/ocr/TesseractOCRParser.java | 11 +-
.../tika/parser/odf/OpenDocumentContentParser.java | 14 +-
.../java/org/apache/tika/parser/xml/XMLParser.java | 9 +-
.../src/test/java/org/apache/tika/TestParsers.java | 2 +-
.../tika/detect/TestContainerAwareDetector.java | 46 ++++-
.../apache/tika/parser/chm/TestChmExtraction.java | 6 +-
25 files changed, 661 insertions(+), 120 deletions(-)
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.
[tika] 01/05: TIKA-2645 for tika-core
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit c203ef3c8b850900fc885badab9390d7e5540440
Author: tballison <ta...@mitre.org>
AuthorDate: Mon May 21 13:10:40 2018 -0400
TIKA-2645 for tika-core
---
.../org/apache/tika/detect/XmlRootExtractor.java | 23 +--
.../java/org/apache/tika/mime/MimeTypesReader.java | 113 +++++++++-
.../java/org/apache/tika/parser/ParseContext.java | 33 +++
.../java/org/apache/tika/utils/XMLReaderUtils.java | 129 ++++++++++--
.../org/apache/tika/MultiThreadedTikaTest.java | 230 +++++++++++++++++----
tika-parsers/pom.xml | 7 +-
.../src/test/java/org/apache/tika/TestParsers.java | 2 +-
.../tika/detect/TestContainerAwareDetector.java | 38 +++-
.../apache/tika/parser/chm/TestChmExtraction.java | 6 +-
9 files changed, 495 insertions(+), 86 deletions(-)
diff --git a/tika-core/src/main/java/org/apache/tika/detect/XmlRootExtractor.java b/tika-core/src/main/java/org/apache/tika/detect/XmlRootExtractor.java
index 74d994d..63c0edd 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/XmlRootExtractor.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/XmlRootExtractor.java
@@ -19,15 +19,15 @@ package org.apache.tika.detect;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
-import javax.xml.XMLConstants;
import javax.xml.namespace.QName;
-import javax.xml.parsers.SAXParserFactory;
+import javax.xml.parsers.SAXParser;
+import org.apache.tika.exception.TikaException;
import org.apache.tika.io.CloseShieldInputStream;
import org.apache.tika.sax.OfflineContentHandler;
+import org.apache.tika.utils.XMLReaderUtils;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
-import org.xml.sax.SAXNotRecognizedException;
import org.xml.sax.helpers.DefaultHandler;
/**
@@ -47,22 +47,15 @@ public class XmlRootExtractor {
*/
public QName extractRootElement(InputStream stream) {
ExtractorHandler handler = new ExtractorHandler();
+ SAXParser parser = null;
try {
- SAXParserFactory factory = SAXParserFactory.newInstance();
- factory.setNamespaceAware(true);
- factory.setValidating(false);
- try {
- factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
- } catch (SAXNotRecognizedException e) {
- // TIKA-271 and TIKA-1000: Some XML parsers do not support the secure-processing
- // feature, even though it's required by JAXP in Java 5. Ignoring
- // the exception is fine here, deployments without this feature
- // are inherently vulnerable to XML denial-of-service attacks.
- }
- factory.newSAXParser().parse(
+ parser = XMLReaderUtils.acquireSAXParser();
+ parser.parse(
new CloseShieldInputStream(stream),
new OfflineContentHandler(handler));
} catch (Exception ignore) {
+ } finally {
+ XMLReaderUtils.releaseParser(parser);
}
return handler.rootElement;
}
diff --git a/tika-core/src/main/java/org/apache/tika/mime/MimeTypesReader.java b/tika-core/src/main/java/org/apache/tika/mime/MimeTypesReader.java
index abd8cb2..b9953b0 100644
--- a/tika-core/src/main/java/org/apache/tika/mime/MimeTypesReader.java
+++ b/tika-core/src/main/java/org/apache/tika/mime/MimeTypesReader.java
@@ -33,7 +33,11 @@ import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.tika.exception.TikaException;
import org.w3c.dom.Document;
import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
@@ -99,7 +103,22 @@ import org.xml.sax.helpers.DefaultHandler;
* @see <a href="http://freedesktop.org/wiki/Standards_2fshared_2dmime_2dinfo_2dspec">http://freedesktop.org/wiki/Standards_2fshared_2dmime_2dinfo_2dspec</a>
*/
public class MimeTypesReader extends DefaultHandler implements MimeTypesReaderMetKeys {
- protected final MimeTypes types;
+ /**
+ * Parser pool size
+ */
+ private static int POOL_SIZE = 10;
+
+ private static final ReentrantReadWriteLock READ_WRITE_LOCK = new ReentrantReadWriteLock();
+
+ private static ArrayBlockingQueue<SAXParser> SAX_PARSERS = new ArrayBlockingQueue<>(POOL_SIZE);
+
+ static {
+ try {
+ setPoolSize(POOL_SIZE);
+ } catch (TikaException e) {
+ throw new RuntimeException("problem initializing SAXParser pool", e);
+ }
+ } protected final MimeTypes types;
/** Current type */
protected MimeType type = null;
@@ -113,23 +132,24 @@ public class MimeTypesReader extends DefaultHandler implements MimeTypesReaderMe
}
public void read(InputStream stream) throws IOException, MimeTypeException {
+ SAXParser parser = null;
try {
- SAXParserFactory factory = SAXParserFactory.newInstance();
- factory.setNamespaceAware(false);
- factory.setFeature(
- XMLConstants.FEATURE_SECURE_PROCESSING, true);
- SAXParser parser = factory.newSAXParser();
+
+ parser = acquireSAXParser();
parser.parse(stream, this);
- } catch (ParserConfigurationException e) {
+ } catch (TikaException e) {
throw new MimeTypeException("Unable to create an XML parser", e);
} catch (SAXException e) {
throw new MimeTypeException("Invalid type configuration", e);
+ } finally {
+ releaseParser(parser);
}
}
public void read(Document document) throws MimeTypeException {
try {
TransformerFactory factory = TransformerFactory.newInstance();
+ factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
Transformer transformer = factory.newTransformer();
transformer.transform(new DOMSource(document), new SAXResult(this));
} catch (TransformerException e) {
@@ -291,5 +311,84 @@ public class MimeTypesReader extends DefaultHandler implements MimeTypesReaderMe
}
}
+ /**
+ * Acquire a SAXParser from the pool; create one if it
+ * doesn't exist. Make sure to {@link #releaseParser(SAXParser)} in
+ * a <code>finally</code> block every time you call this.
+ *
+ * @return a SAXParser
+ * @throws TikaException
+ */
+ public static SAXParser acquireSAXParser()
+ throws TikaException {
+ while (true) {
+ SAXParser parser = null;
+ try {
+ READ_WRITE_LOCK.readLock().lock();
+ parser = SAX_PARSERS.poll(10, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ throw new TikaException("interrupted while waiting for SAXParser", e);
+ } finally {
+ READ_WRITE_LOCK.readLock().unlock();
+ }
+ if (parser != null) {
+ return parser;
+ }
+ }
+ }
+
+ /**
+ * Return parser to the pool for reuse
+ *
+ * @param parser parser to return
+ */
+ public static void releaseParser(SAXParser parser) {
+ try {
+ parser.reset();
+ } catch (UnsupportedOperationException e) {
+ //ignore
+ }
+ try {
+ READ_WRITE_LOCK.readLock().lock();
+ //if there are extra parsers (e.g. after a reset of the pool to a smaller size),
+ // this parser will not be added and will then be gc'd
+ SAX_PARSERS.offer(parser);
+ } finally {
+ READ_WRITE_LOCK.readLock().unlock();
+ }
+ }
+
+ /**
+ * Set the pool size for cached XML parsers.
+ *
+ * @param poolSize
+ */
+ public static void setPoolSize(int poolSize) throws TikaException {
+ try {
+ //stop the world with a write lock
+ //parsers that are currently in use will be offered, but not
+ //accepted and will be gc'd
+ READ_WRITE_LOCK.writeLock().lock();
+ SAX_PARSERS = new ArrayBlockingQueue<>(poolSize);
+ for (int i = 0; i < poolSize; i++) {
+ SAX_PARSERS.offer(newSAXParser());
+ }
+ POOL_SIZE = poolSize;
+ } finally {
+ READ_WRITE_LOCK.writeLock().unlock();
+ }
+ }
+
+ private static SAXParser newSAXParser() throws TikaException {
+ SAXParserFactory factory = SAXParserFactory.newInstance();
+ factory.setNamespaceAware(false);
+ try {
+ factory.setFeature(
+ XMLConstants.FEATURE_SECURE_PROCESSING, true);
+ return factory.newSAXParser();
+ } catch (ParserConfigurationException|SAXException e) {
+ throw new TikaException("prooblem creating SAX parser factory", e);
+ }
+ }
}
diff --git a/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java b/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
index 68df47b..2a3afcd 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
@@ -52,6 +52,7 @@ public class ParseContext implements Serializable {
/** Serial version UID. */
private static final long serialVersionUID = -5921436862145826534L;
+ private final XMLReaderUtils xmlReaderUtils = new XMLReaderUtils();
/** Map of objects in this context */
private final Map<String, Object> context = new HashMap<String, Object>();
@@ -138,6 +139,38 @@ public class ParseContext implements Serializable {
}
/**
+ * Returns the SAX parser specified in this parsing context. If a parser
+ * is not explicitly specified, then one is acquired from the pool.
+ * <p>
+ * Make sure to {@link #releaseParser(SAXParser)} in
+ * a <code>finally</code> block every time you call this.
+ * </p>
+ *
+ * @return SAXParser
+ * @throws TikaException
+ */
+ public SAXParser acquireSAXParser() throws TikaException {
+ if (context.containsKey(SAXParser.class)) {
+ return get(SAXParser.class);
+ }
+ return xmlReaderUtils.acquireSAXParser();
+ }
+
+ /**
+ * If the context already has a SAXParser, this is a no-op.
+ * Otherwise, this returns the parser to the pool
+ *
+ * @param parser
+ * @throws TikaException
+ */
+ public void releaseParser(SAXParser parser) throws TikaException {
+ if (context.containsKey(SAXParser.class)) {
+ return;
+ }
+ xmlReaderUtils.releaseParser(parser);
+ }
+
+ /**
* Returns the SAX parser factory specified in this parsing context.
* If a factory is not explicitly specified, then a default factory
* instance is created and returned. The default factory instance is
diff --git a/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java b/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java
index eeedcc6..4ed58b4 100644
--- a/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java
+++ b/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java
@@ -38,9 +38,12 @@ import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.TransformerFactoryConfigurationError;
-
import java.io.IOException;
+import java.io.Serializable;
import java.io.StringReader;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -49,9 +52,33 @@ import java.util.logging.Logger;
* to use the {@link org.apache.tika.sax.OfflineContentHandler} to guard against
* XML External Entity attacks.
*/
-public class XMLReaderUtils {
+public class XMLReaderUtils implements Serializable {
+
+ /**
+ * Serial version UID
+ */
+ private static final long serialVersionUID = 6110455808615143122L;
+
private static final Logger LOG = Logger.getLogger(XMLReaderUtils.class.getName());
+ /**
+ * Parser pool size
+ */
+ private static int POOL_SIZE = 10;
+
+ private static final ReentrantReadWriteLock READ_WRITE_LOCK = new ReentrantReadWriteLock();
+
+ private static ArrayBlockingQueue<SAXParser> SAX_PARSERS = new ArrayBlockingQueue<>(POOL_SIZE);
+
+ static {
+ try {
+ setPoolSize(POOL_SIZE);
+ } catch (TikaException e) {
+ throw new RuntimeException("problem initializing SAXParser pool", e);
+ }
+ }
+
+
private static final EntityResolver IGNORING_SAX_ENTITY_RESOLVER = new EntityResolver() {
public InputSource resolveEntity(String publicId, String systemId) throws SAXException, IOException {
return new InputSource(new StringReader(""));
@@ -72,10 +99,10 @@ public class XMLReaderUtils {
* is not explicitly specified, then one is created using the specified
* or the default SAX parser.
*
- * @see #getSAXParser()
- * @since Apache Tika 1.13
* @return XMLReader
* @throws TikaException
+ * @see #getSAXParser()
+ * @since Apache Tika 1.13
*/
public static XMLReader getXMLReader() throws TikaException {
XMLReader reader;
@@ -96,12 +123,11 @@ public class XMLReaderUtils {
* Make sure to wrap your handler in the {@link org.apache.tika.sax.OfflineContentHandler} to
* prevent XML External Entity attacks
* </p>
-
*
- * @see #getSAXParserFactory()
- * @since Apache Tika 0.8
* @return SAX parser
* @throws TikaException if a SAX parser could not be created
+ * @see #getSAXParserFactory()
+ * @since Apache Tika 0.8
*/
public static SAXParser getSAXParser() throws TikaException {
try {
@@ -124,8 +150,8 @@ public class XMLReaderUtils {
* prevent XML External Entity attacks
* </p>
*
- * @since Apache Tika 0.8
* @return SAX parser factory
+ * @since Apache Tika 0.8
*/
public static SAXParserFactory getSAXParserFactory() {
SAXParserFactory factory = SAXParserFactory.newInstance();
@@ -154,8 +180,8 @@ public class XMLReaderUtils {
* configured to be namespace-aware and to apply reasonable security
* features.
*
- * @since Apache Tika 1.13
* @return DOM parser factory
+ * @since Apache Tika 1.13
*/
public static DocumentBuilderFactory getDocumentBuilderFactory() {
//borrowed from Apache POI
@@ -179,8 +205,8 @@ public class XMLReaderUtils {
* configured to apply an {@link #IGNORING_SAX_ENTITY_RESOLVER},
* and it sets the ErrorHandler to <code>null</code>.
*
- * @since Apache Tika 1.13
* @return DOM Builder
+ * @since Apache Tika 1.13
*/
public static DocumentBuilder getDocumentBuilder() throws TikaException {
try {
@@ -201,8 +227,8 @@ public class XMLReaderUtils {
* configured to be namespace-aware and to apply reasonable security
* using the {@link #IGNORING_STAX_ENTITY_RESOLVER}.
*
- * @since Apache Tika 1.13
* @return StAX input factory
+ * @since Apache Tika 1.13
*/
public static XMLInputFactory getXMLInputFactory() {
XMLInputFactory factory = XMLInputFactory.newFactory();
@@ -218,9 +244,9 @@ public class XMLReaderUtils {
try {
documentBuilderFactory.setFeature(feature, enabled);
} catch (Exception e) {
- LOG.log(Level.WARNING, "SAX Feature unsupported: "+feature, e);
+ LOG.log(Level.WARNING, "SAX Feature unsupported: " + feature, e);
} catch (AbstractMethodError ame) {
- LOG.log(Level.WARNING, "Cannot set SAX feature because outdated XML parser in classpath: "+ feature, ame);
+ LOG.log(Level.WARNING, "Cannot set SAX feature because outdated XML parser in classpath: " + feature, ame);
}
}
@@ -234,13 +260,13 @@ public class XMLReaderUtils {
/**
* Returns a new transformer
- *
+ * <p>
* The transformer instance is configured to to use
* {@link XMLConstants#FEATURE_SECURE_PROCESSING secure XML processing}.
*
- * @since Apache Tika 1.17
* @return Transformer
* @throws TikaException when the transformer can not be created
+ * @since Apache Tika 1.17
*/
public static Transformer getTransformer() throws TikaException {
try {
@@ -249,7 +275,78 @@ public class XMLReaderUtils {
return transformerFactory.newTransformer();
} catch (TransformerConfigurationException | TransformerFactoryConfigurationError e) {
throw new TikaException("Transformer not available", e);
- }
+ }
}
+ /**
+ * Acquire a SAXParser from the pool; create one if it
+ * doesn't exist. Make sure to {@link #releaseParser(SAXParser)} in
+ * a <code>finally</code> block every time you call this.
+ *
+ * @return a SAXParser
+ * @throws TikaException
+ */
+ public static SAXParser acquireSAXParser()
+ throws TikaException {
+ while (true) {
+ SAXParser parser = null;
+ try {
+ READ_WRITE_LOCK.readLock().lock();
+ parser = SAX_PARSERS.poll(10, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ throw new TikaException("interrupted while waiting for SAXParser", e);
+ } finally {
+ READ_WRITE_LOCK.readLock().unlock();
+
+ }
+ if (parser != null) {
+ return parser;
+ }
+ }
+ }
+
+ /**
+ * Return parser to the pool for reuse
+ *
+ * @param parser parser to return
+ */
+ public static void releaseParser(SAXParser parser) {
+ try {
+ parser.reset();
+ } catch (UnsupportedOperationException e) {
+ //ignore
+ }
+ try {
+ READ_WRITE_LOCK.readLock().lock();
+ //if there are extra parsers (e.g. after a reset of the pool to a smaller size),
+ // this parser will not be added and will then be gc'd
+ boolean success = SAX_PARSERS.offer(parser);
+ } finally {
+ READ_WRITE_LOCK.readLock().unlock();
+ }
+ }
+
+ /**
+ * Set the pool size for cached XML parsers.
+ *
+ * @param poolSize
+ */
+ public static void setPoolSize(int poolSize) throws TikaException {
+ try {
+ //stop the world with a write lock.
+ //parsers that are currently in use will be offered, but not
+ //accepted and will be gc'd
+ READ_WRITE_LOCK.writeLock().lock();
+ if (SAX_PARSERS.size() == poolSize) {
+ return;
+ }
+ SAX_PARSERS = new ArrayBlockingQueue<>(poolSize);
+ for (int i = 0; i < poolSize; i++) {
+ SAX_PARSERS.offer(getSAXParser());
+ }
+ POOL_SIZE = poolSize;
+ } finally {
+ READ_WRITE_LOCK.writeLock().unlock();
+ }
+ }
}
diff --git a/tika-core/src/test/java/org/apache/tika/MultiThreadedTikaTest.java b/tika-core/src/test/java/org/apache/tika/MultiThreadedTikaTest.java
index bd0f263..3d5094b 100644
--- a/tika-core/src/test/java/org/apache/tika/MultiThreadedTikaTest.java
+++ b/tika-core/src/test/java/org/apache/tika/MultiThreadedTikaTest.java
@@ -17,7 +17,12 @@
package org.apache.tika;
+import org.apache.tika.detect.Detector;
+import org.apache.tika.detect.XmlRootExtractor;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
@@ -25,9 +30,9 @@ import org.apache.tika.parser.RecursiveParserWrapper;
import org.apache.tika.sax.AbstractRecursiveParserWrapperHandler;
import org.apache.tika.sax.BasicContentHandlerFactory;
import org.apache.tika.sax.RecursiveParserWrapperHandler;
-import org.junit.Test;
-import org.xml.sax.helpers.DefaultHandler;
+import org.apache.tika.utils.XMLReaderUtils;
+import javax.xml.namespace.QName;
import java.io.FileFilter;
import java.io.IOException;
import java.io.InputStream;
@@ -40,7 +45,6 @@ import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
-import java.util.Locale;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.Callable;
@@ -57,41 +61,106 @@ import static org.junit.Assert.assertEquals;
public class MultiThreadedTikaTest extends TikaTest {
//TODO: figure out how to make failures reproducible a la Lucene/Solr with a seed
//TODO: Consider randomizing the Locale and timezone, like Lucene/Solr...
+ XmlRootExtractor ex = new XmlRootExtractor();
/**
- * This calls {@link #testEach(Path[], int, int)} and then {@link #testAll(Path[], int, int)}
+ * This calls {@link #testEach(Path[], ParseContext[], int, int)} and then {@link #testAll(Path[], ParseContext[], int, int)}
*
- * @param numThreads number of threads to use
+ * @param numThreads number of threads to use
* @param numIterations number of iterations per thread
- * @param filter file filter to select files from "/test-documents"; if <code>null</code>,
- * all files will be used
+ * @param filter file filter to select files from "/test-documents"; if <code>null</code>,
+ * all files will be used
* @throws Exception
*/
- protected void testMultiThreaded(int numThreads, int numIterations, FileFilter filter) throws Exception {
+ protected void testMultiThreaded(ParseContext[] parseContext, int numThreads, int numIterations, FileFilter filter) throws Exception {
Path[] allFiles = getTestFiles(filter);
- //testEach(allFiles, numThreads, numIterations);
- testAll(allFiles, numThreads, numIterations);
+ testEach(allFiles, parseContext, numThreads, numIterations);
+ testAll(allFiles, parseContext, numThreads, numIterations);
+ }
+
+ public void testDetector(Detector detector, int numThreads, int numIterations, FileFilter filter, int randomlyResizeSAXPool) throws Exception {
+ Path[] files = getTestFiles(filter);
+ testDetectorEach(detector, files, numThreads, numIterations, randomlyResizeSAXPool);
+ testDetectorOnAll(detector, files, numThreads, numIterations, randomlyResizeSAXPool);
+ }
+
+ void testDetectorEach(Detector detector, Path[] files, int numThreads, int numIterations, int randomlyResizeSAXPool) {
+ for (Path p : files) {
+ Path[] toTest = new Path[1];
+ toTest[0] = p;
+ testDetectorOnAll(detector, toTest, numThreads, numIterations, randomlyResizeSAXPool);
+ }
+ }
+
+ private void testDetectorOnAll(Detector detector, Path[] toTest, int numThreads, int numIterations, int randomlyResizeSAXPool) {
+ Map<Path, MediaType> truth = getBaselineDetection(detector, toTest);
+ //if all files caused an exception
+ if (truth.size() == 0) {
+ return;
+ }
+ //only those that parsed without exception
+ Path[] testFiles = new Path[truth.size()];
+ int j = 0;
+ for (Path testFile : truth.keySet()) {
+ testFiles[j++] = testFile;
+ }
+ int actualThreadCount = numThreads + ((randomlyResizeSAXPool > 0) ? randomlyResizeSAXPool : 0);
+ ExecutorService ex = Executors.newFixedThreadPool(actualThreadCount);
+ try {
+ _testDetectorOnAll(detector, testFiles, numThreads, numIterations, truth, ex, randomlyResizeSAXPool);
+ } finally {
+ ex.shutdown();
+ ex.shutdownNow();
+ }
+ }
+
+ private void _testDetectorOnAll(Detector detector, Path[] testFiles, int numThreads,
+ int numIterations, Map<Path, MediaType> truth, ExecutorService ex, int randomlyResizeSAXPool) {
+ ExecutorCompletionService<Integer> executorCompletionService = new ExecutorCompletionService<>(ex);
+
+ executorCompletionService.submit(new SAXPoolResizer(randomlyResizeSAXPool));
+ for (int i = 0; i < numThreads; i++) {
+ executorCompletionService.submit(new TikaDetectorRunner(detector, numIterations, testFiles, truth));
+ }
+
+ int completed = 0;
+ while (completed < numThreads) {
+ //TODO: add a maximum timeout threshold
+
+ Future<Integer> future = null;
+ try {
+ future = executorCompletionService.poll(1000, TimeUnit.MILLISECONDS);
+ if (future != null) {
+ future.get();//trigger exceptions from thread
+ completed++;
+ }
+ } catch (InterruptedException | ExecutionException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ ex.shutdown();
+ ex.shutdownNow();
}
/**
- * Test each file, one at a time in multiple threads.
- * This was required to test TIKA-2519 in a reasonable
- * amount of time. This forced the parser to use the
- * same underlying memory structurees because it was the same file.
- * This is stricter than I think our agreement with clients is
- * because this run tests on literally the same file and
- * not a copy of the file per thread. Let's leave this as is
- * unless there's a good reason to create a separate copy per thread.
+ * Test each file, one at a time in multiple threads.
+ * This was required to test TIKA-2519 in a reasonable
+ * amount of time. This forced the parser to use the
+ * same underlying memory structures because it was the same file.
+ * This is stricter than I think our agreement with clients is
+ * because this run tests on literally the same file and
+ * not a copy of the file per thread. Let's leave this as is
+ * unless there's a good reason to create a separate copy per thread.
*
- * @param files files to test, one at a time
- * @param numThreads number of threads to use
+ * @param files files to test, one at a time
+ * @param numThreads number of threads to use
* @param numIterations number of iterations per thread
*/
- protected void testEach(Path[] files, int numThreads, int numIterations) {
+ protected void testEach(Path[] files, ParseContext[] parseContext, int numThreads, int numIterations) {
for (Path p : files) {
Path[] toTest = new Path[1];
toTest[0] = p;
- testAll(toTest, numThreads, numIterations);
+ testAll(toTest, parseContext, numThreads, numIterations);
}
}
@@ -99,18 +168,18 @@ public class MultiThreadedTikaTest extends TikaTest {
* This tests all files together. Each parser randomly selects
* a file from the array. Two parsers could wind up parsing the
* same file at the same time. Good.
- *
+ * <p>
* In the current implementation, this gets ground truth only
* from files that do not throw exceptions. This will ignore
* files that cause exceptions.
*
- * @param files files to parse
- * @param numThreads number of parser threads
+ * @param files files to parse
+ * @param numThreads number of parser threads
* @param numIterations number of iterations per parser
*/
- protected void testAll(Path[] files, int numThreads, int numIterations) {
+ protected void testAll(Path[] files, ParseContext[] parseContext, int numThreads, int numIterations) {
- Map<Path, Extract> truth = getBaseline(files);
+ Map<Path, Extract> truth = getBaseline(files, parseContext[0]);
//if all files caused an exception
if (truth.size() == 0) {
return;
@@ -124,24 +193,23 @@ public class MultiThreadedTikaTest extends TikaTest {
ExecutorService ex = Executors.newFixedThreadPool(numThreads);
try {
- _testAll(testFiles, numThreads, numIterations, truth, ex);
+ _testAll(testFiles, parseContext, numThreads, numIterations, truth, ex);
} finally {
ex.shutdown();
ex.shutdownNow();
}
}
- private void _testAll(Path[] testFiles, int numThreads, int numIterations,
+ private void _testAll(Path[] testFiles, ParseContext[] parseContext, int numThreads, int numIterations,
Map<Path, Extract> truth, ExecutorService ex) {
ExecutorCompletionService<Integer> executorCompletionService = new ExecutorCompletionService<>(ex);
//use the same parser in all threads
Parser parser = new AutoDetectParser();
- RecursiveParserWrapper wrapper = new RecursiveParserWrapper(parser,
- new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT, -1));
+ RecursiveParserWrapper wrapper = new RecursiveParserWrapper(parser);
for (int i = 0; i < numThreads; i++) {
- executorCompletionService.submit(new TikaRunner(wrapper, numIterations, testFiles, truth));
+ executorCompletionService.submit(new TikaRunner(wrapper, parseContext[i], numIterations, testFiles, truth));
}
int completed = 0;
@@ -155,20 +223,20 @@ public class MultiThreadedTikaTest extends TikaTest {
future.get();//trigger exceptions from thread
completed++;
}
- } catch (InterruptedException|ExecutionException e) {
+ } catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
}
}
- private static Path[] getTestFiles(final FileFilter fileFilter) throws URISyntaxException, IOException {
+ public static Path[] getTestFiles(final FileFilter fileFilter) throws URISyntaxException, IOException {
Path root = Paths.get(
MultiThreadedTikaTest.class.getResource("/test-documents").toURI());
final List<Path> files = new ArrayList<>();
Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
- if (fileFilter != null && ! fileFilter.accept(file.toFile())) {
+ if (fileFilter != null && !fileFilter.accept(file.toFile())) {
return FileVisitResult.CONTINUE;
}
if (!attrs.isDirectory()) {
@@ -182,7 +250,23 @@ public class MultiThreadedTikaTest extends TikaTest {
return files.toArray(new Path[files.size()]);
}
- private static ConcurrentHashMap<Path, Extract> getBaseline(Path[] files) {
+ private static ConcurrentHashMap<Path, MediaType> getBaselineDetection(Detector detector, Path[] files) {
+
+ ConcurrentHashMap<Path, MediaType> baseline = new ConcurrentHashMap<>();
+ XmlRootExtractor extractor = new XmlRootExtractor();
+ for (Path f : files) {
+ Metadata metadata = new Metadata();
+ try (TikaInputStream tis = TikaInputStream.get(f, metadata)) {
+ baseline.put(f, detector.detect(tis, metadata));
+ baseline.put(f, detector.detect(tis, metadata));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ return baseline;
+ }
+
+ private static ConcurrentHashMap<Path, Extract> getBaseline(Path[] files, ParseContext parseContext) {
ConcurrentHashMap<Path, Extract> baseline = new ConcurrentHashMap<>();
for (Path f : files) {
@@ -192,8 +276,8 @@ public class MultiThreadedTikaTest extends TikaTest {
RecursiveParserWrapperHandler handler = new RecursiveParserWrapperHandler(
new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT, -1),
-1);
- try (InputStream is = Files.newInputStream(f)) {
- wrapper.parse(is, handler, new Metadata(), new ParseContext());
+ try (TikaInputStream is = TikaInputStream.get(f)) {
+ wrapper.parse(is, handler, new Metadata(), parseContext);
}
List<Metadata> metadataList = handler.getMetadataList();
baseline.put(f, new Extract(metadataList));
@@ -204,7 +288,8 @@ public class MultiThreadedTikaTest extends TikaTest {
return baseline;
}
- private static List<Metadata> getRecursiveMetadata(InputStream is, RecursiveParserWrapper wrapper) throws Exception {
+ private static List<Metadata> getRecursiveMetadata(InputStream is,
+ RecursiveParserWrapper wrapper, ParseContext parseContext) throws Exception {
//different from parent TikaTest in that this extracts text.
//can't extract xhtml because "tmp" file names wind up in
//content's metadata and they'll differ by file.
@@ -212,24 +297,81 @@ public class MultiThreadedTikaTest extends TikaTest {
RecursiveParserWrapperHandler handler = new RecursiveParserWrapperHandler(
new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT, -1),
-1);
- wrapper.parse(is, handler, new Metadata(), new ParseContext());
+ wrapper.parse(is, handler, new Metadata(), parseContext);
return handler.getMetadataList();
}
+ private class SAXPoolResizer implements Callable<Integer> {
+ private final int maxResize;
+ private final Random rand = new Random();
+ SAXPoolResizer(int maxResize) {
+ this.maxResize = maxResize;
+ }
+
+ public Integer call() throws TikaException {
+ int resized = 0;
+ while (true) {
+ try {
+ Thread.yield();
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ return resized;
+ }
+ if (maxResize > 0 && rand.nextFloat() > 0.01) {
+ int sz = rand.nextInt(maxResize)+1;
+ XMLReaderUtils.setPoolSize(sz);
+ resized++;
+ }
+ }
+ }
+ }
+
+ private class TikaDetectorRunner implements Callable<Integer> {
+ private final Detector detector;
+ private final int iterations;
+ private final Path[] files;
+ private final Map<Path, MediaType> truth;
+ private final Random random = new Random();
+
+ private TikaDetectorRunner(Detector detector, int iterations, Path[] files, Map<Path, MediaType> truth) {
+ this.detector = detector;
+ this.iterations = iterations;
+ this.files = files;
+ this.truth = truth;
+ }
+
+ @Override
+ public Integer call() throws Exception {
+ for (int i = 0; i < iterations; i++) {
+ int randIndex = random.nextInt(files.length);
+ Path testFile = files[randIndex];
+ Metadata metadata = new Metadata();
+ try (TikaInputStream tis = TikaInputStream.get(testFile, metadata)) {
+ MediaType mediaType = detector.detect(tis, metadata);
+ assertEquals("failed on: " + testFile.getFileName(), truth.get(testFile), mediaType);
+ }
+ }
+ return 1;
+ }
+
+ }
+
+
//TODO: make this return something useful besides an integer
private class TikaRunner implements Callable<Integer> {
private final RecursiveParserWrapper parser;
private final int iterations;
private final Path[] files;
private final Map<Path, Extract> truth;
-
+ private final ParseContext parseContext;
private final Random random = new Random();
- private TikaRunner(RecursiveParserWrapper parser, int iterations, Path[] files, Map<Path, Extract> truth) {
+ private TikaRunner(RecursiveParserWrapper parser, ParseContext parseContext, int iterations, Path[] files, Map<Path, Extract> truth) {
this.parser = parser;
this.iterations = iterations;
this.files = files;
this.truth = truth;
+ this.parseContext = parseContext;
}
@Override
@@ -238,10 +380,10 @@ public class MultiThreadedTikaTest extends TikaTest {
int randIndex = random.nextInt(files.length);
Path testFile = files[randIndex];
try (InputStream is = Files.newInputStream(testFile)) {
- List<Metadata> metadataList = getRecursiveMetadata(is, parser);
+ List<Metadata> metadataList = getRecursiveMetadata(is, parser, parseContext);
assertExtractEquals(truth.get(testFile), new Extract(metadataList));
} catch (Exception e) {
- throw new RuntimeException(testFile+" triggered this exception", e);
+ throw new RuntimeException(testFile + " triggered this exception", e);
}
}
return 1;
diff --git a/tika-parsers/pom.xml b/tika-parsers/pom.xml
index 546a7ef..fc1a028 100644
--- a/tika-parsers/pom.xml
+++ b/tika-parsers/pom.xml
@@ -732,7 +732,12 @@
</exclusion>
</exclusions>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>3.6.1</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/tika-parsers/src/test/java/org/apache/tika/TestParsers.java b/tika-parsers/src/test/java/org/apache/tika/TestParsers.java
index 173555b..e359fbc 100644
--- a/tika-parsers/src/test/java/org/apache/tika/TestParsers.java
+++ b/tika-parsers/src/test/java/org/apache/tika/TestParsers.java
@@ -113,6 +113,6 @@ public class TestParsers extends MultiThreadedTikaTest {
@Ignore("ignore for regular builds; run occasionally")
public void testAllMultiThreaded() throws Exception {
//this runs against all files in /test-documents
- testMultiThreaded(10, 100, null);
+ //testMultiThreaded(10, 100, null);
}
}
diff --git a/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java b/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java
index e4117c4..6828342 100644
--- a/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java
+++ b/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java
@@ -21,23 +21,28 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.File;
+import java.io.FileFilter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.file.Path;
import org.apache.poi.poifs.filesystem.NPOIFSFileSystem;
+import org.apache.tika.MultiThreadedTikaTest;
+import org.apache.tika.Tika;
import org.apache.tika.config.TikaConfig;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
import org.apache.tika.mime.MimeTypes;
import org.apache.tika.parser.iwork.iwana.IWork13PackageParser;
+import org.apache.tika.utils.XMLReaderUtils;
import org.junit.Test;
/**
* Junit test class for {@link ContainerAwareDetector}
*/
-public class TestContainerAwareDetector {
+public class TestContainerAwareDetector extends MultiThreadedTikaTest {
private final TikaConfig tikaConfig = TikaConfig.getDefaultConfig();
private final MimeTypes mimeTypes = tikaConfig.getMimeRepository();
private final Detector detector = new DefaultDetector(mimeTypes);
@@ -435,4 +440,35 @@ public class TestContainerAwareDetector {
}
}
+ @Test
+ public void testXMLMultiThreaded() throws Exception {
+ Detector detector = new Tika().getDetector();
+ FileFilter filter = new FileFilter() {
+ @Override
+ public boolean accept(File pathname) {
+ if (pathname.getName().endsWith(".xml")) {
+ return true;
+ }
+ return false;
+ }
+ };
+ int numThreads = 1;
+ XMLReaderUtils.setPoolSize(numThreads);
+ testDetector(detector, numThreads, 20, filter, numThreads*2);
+ }
+
+ @Test
+ public void testAllMultithreaded() throws Exception {
+ Detector detector = new Tika().getDetector();
+ FileFilter filter = new FileFilter() {
+ @Override
+ public boolean accept(File pathname) {
+ return true;
+ }
+ };
+ int numThreads = 20;
+ XMLReaderUtils.setPoolSize(numThreads);
+ testDetector(detector, numThreads, 50, filter, numThreads*3);
+ }
+
}
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/chm/TestChmExtraction.java b/tika-parsers/src/test/java/org/apache/tika/parser/chm/TestChmExtraction.java
index 90fe125..3129453 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/chm/TestChmExtraction.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/chm/TestChmExtraction.java
@@ -214,7 +214,11 @@ public class TestChmExtraction extends MultiThreadedTikaTest {
@Test
public void testMultiThreaded() throws Exception {
- testMultiThreaded(10, 10, new FileFilter() {
+ ParseContext[] parseContexts = new ParseContext[10];
+ for (int i = 0; i < parseContexts.length; i++) {
+ parseContexts[i] = new ParseContext();
+ }
+ testMultiThreaded(parseContexts, 10, 10, new FileFilter() {
@Override
public boolean accept(File pathname) {
if (pathname.getName().toLowerCase(Locale.ENGLISH).endsWith(".chm")) {
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.
[tika] 05/05: update changes
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit c40045ab6c9ea51f7480f46424e65a1a4c947038
Author: tballison <ta...@mitre.org>
AuthorDate: Mon May 21 14:31:36 2018 -0400
update changes
---
CHANGES.txt | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/CHANGES.txt b/CHANGES.txt
index c0925fa..d94eaa7 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,13 @@
+Release 1.19 ???
+
+
+ * Use a pool for SAXParsers rather than creating a new one for every parse.
+ For better performance, set XMLReaderUtils.setPoolSize() to the
+ number of threads you're using with Tika (TIKA-2645.
+
+ * Add the RecursiveParserWrapperHandler to improve the RecursiveParserWrapper
+ API slightly (TIKA-2644).
+
Release 1.18 - 4/20/2018
* Upgrade Jackson to 2.9.5 (TIKA-2634).
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.
[tika] 02/05: TIKA-2645 - use a pool for SAXParsers -- tika-parsers
package
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 0101164e1c525191da9a571b97739bf289ea32b4
Author: tballison <ta...@mitre.org>
AuthorDate: Mon May 21 13:55:56 2018 -0400
TIKA-2645 - use a pool for SAXParsers -- tika-parsers package
---
.../java/org/apache/tika/parser/NetworkParser.java | 7 ++++++-
.../java/org/apache/tika/parser/ParseContext.java | 7 +++----
.../java/org/apache/tika/utils/XMLReaderUtils.java | 14 ++++++++++++--
.../java/org/apache/tika/parser/dif/DIFParser.java | 7 ++++++-
.../apache/tika/parser/epub/EpubContentParser.java | 14 ++++++++++----
.../apache/tika/parser/iwork/IWorkPackageParser.java | 16 ++++++++++++----
.../microsoft/ooxml/AbstractOOXMLExtractor.java | 8 +++++++-
.../ooxml/SXSLFPowerPointExtractorDecorator.java | 14 ++++++++++++--
.../microsoft/ooxml/SXWPFWordExtractorDecorator.java | 9 +++++++--
.../microsoft/ooxml/XSSFExcelExtractorDecorator.java | 12 ++++++++++--
.../microsoft/ooxml/xps/XPSExtractorDecorator.java | 20 ++++++++++++++++----
.../parser/microsoft/ooxml/xwpf/XWPFStylesShim.java | 12 ++++++++++--
.../ooxml/xwpf/ml2006/Word2006MLParser.java | 8 ++++++--
.../parser/microsoft/xml/AbstractXML2003Parser.java | 7 ++++++-
.../apache/tika/parser/ocr/TesseractOCRParser.java | 11 +++++++++--
.../tika/parser/odf/OpenDocumentContentParser.java | 14 +++++++++-----
.../java/org/apache/tika/parser/xml/XMLParser.java | 7 ++++++-
.../tika/detect/TestContainerAwareDetector.java | 10 +++++++++-
18 files changed, 156 insertions(+), 41 deletions(-)
diff --git a/tika-core/src/main/java/org/apache/tika/parser/NetworkParser.java b/tika-core/src/main/java/org/apache/tika/parser/NetworkParser.java
index 0f338b7..78b7246 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/NetworkParser.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/NetworkParser.java
@@ -42,6 +42,8 @@ import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
+import javax.xml.parsers.SAXParser;
+
public class NetworkParser extends AbstractParser {
private final URI uri;
@@ -124,8 +126,10 @@ public class NetworkParser extends AbstractParser {
TaggedContentHandler tagged = new TaggedContentHandler(
new OfflineContentHandler(handler));
+ SAXParser parser = null;
try {
- context.getSAXParser().parse(
+ parser = context.acquireSAXParser();
+ parser.parse(
stream, new TeeContentHandler(
tagged, new MetaHandler(metadata)));
} catch (SAXException e) {
@@ -136,6 +140,7 @@ public class NetworkParser extends AbstractParser {
throw new TikaException(
"Unable to read network parser output", e);
} finally {
+ context.releaseParser(parser);
try {
thread.join(1000);
} catch (InterruptedException e) {
diff --git a/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java b/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
index 2a3afcd..6ad85ff 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
@@ -52,7 +52,6 @@ public class ParseContext implements Serializable {
/** Serial version UID. */
private static final long serialVersionUID = -5921436862145826534L;
- private final XMLReaderUtils xmlReaderUtils = new XMLReaderUtils();
/** Map of objects in this context */
private final Map<String, Object> context = new HashMap<String, Object>();
@@ -153,7 +152,7 @@ public class ParseContext implements Serializable {
if (context.containsKey(SAXParser.class)) {
return get(SAXParser.class);
}
- return xmlReaderUtils.acquireSAXParser();
+ return XMLReaderUtils.acquireSAXParser();
}
/**
@@ -163,11 +162,11 @@ public class ParseContext implements Serializable {
* @param parser
* @throws TikaException
*/
- public void releaseParser(SAXParser parser) throws TikaException {
+ public void releaseParser(SAXParser parser) {
if (context.containsKey(SAXParser.class)) {
return;
}
- xmlReaderUtils.releaseParser(parser);
+ XMLReaderUtils.releaseParser(parser);
}
/**
diff --git a/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java b/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java
index 4ed58b4..a2ad9fc 100644
--- a/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java
+++ b/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java
@@ -288,20 +288,26 @@ public class XMLReaderUtils implements Serializable {
*/
public static SAXParser acquireSAXParser()
throws TikaException {
+ int waiting = 0;
while (true) {
SAXParser parser = null;
try {
READ_WRITE_LOCK.readLock().lock();
- parser = SAX_PARSERS.poll(10, TimeUnit.MILLISECONDS);
+ parser = SAX_PARSERS.poll(100, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
throw new TikaException("interrupted while waiting for SAXParser", e);
} finally {
READ_WRITE_LOCK.readLock().unlock();
-
}
if (parser != null) {
return parser;
}
+ waiting++;
+ if (waiting > 3000) {
+ //better to get an exception than have permahang by a bug in one of our parsers
+ throw new TikaException("Waited more than 5 minutes for a SAXParser; this could indicate SAXParser leakage. " +
+ "Please report this to the Tika team dev@tika.apache.org");
+ }
}
}
@@ -321,6 +327,10 @@ public class XMLReaderUtils implements Serializable {
//if there are extra parsers (e.g. after a reset of the pool to a smaller size),
// this parser will not be added and will then be gc'd
boolean success = SAX_PARSERS.offer(parser);
+ if (! success) {
+ LOG.warning("SAXParser not taken back into pool. If you haven't resized the pool, this could " +
+ "be a sign that there are more calls to 'acquire' than to 'release'");
+ }
} finally {
READ_WRITE_LOCK.readLock().unlock();
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/dif/DIFParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/dif/DIFParser.java
index 5957508..237fa0b 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/dif/DIFParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/dif/DIFParser.java
@@ -36,6 +36,8 @@ import org.apache.tika.sax.XHTMLContentHandler;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
+import javax.xml.parsers.SAXParser;
+
public class DIFParser extends AbstractParser {
/**
@@ -61,8 +63,10 @@ public class DIFParser extends AbstractParser {
xhtml.startDocument();
xhtml.startElement("p");
TaggedContentHandler tagged = new TaggedContentHandler(handler);
+ SAXParser parser = null;
try {
- context.getSAXParser().parse(
+ parser = context.acquireSAXParser();
+ parser.parse(
new CloseShieldInputStream(stream),
new OfflineContentHandler(new EmbeddedContentHandler(
getContentHandler(tagged, metadata, context))));
@@ -70,6 +74,7 @@ public class DIFParser extends AbstractParser {
tagged.throwIfCauseOf(e);
throw new TikaException("XML parse error", e);
} finally {
+ context.releaseParser(parser);
xhtml.endElement("p");
xhtml.endDocument();
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/epub/EpubContentParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/epub/EpubContentParser.java
index 83686d9..44d5f72 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/epub/EpubContentParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/epub/EpubContentParser.java
@@ -48,10 +48,16 @@ public class EpubContentParser extends AbstractParser {
Metadata metadata, ParseContext context)
throws IOException, SAXException, TikaException {
- SAXParser parser = context.getSAXParser();
- parser.parse(
- new CloseShieldInputStream(stream),
- new OfflineContentHandler(handler));
+ SAXParser parser = null;
+ try {
+ parser = context.acquireSAXParser();
+
+ parser.parse(
+ new CloseShieldInputStream(stream),
+ new OfflineContentHandler(handler));
+ } finally {
+ context.releaseParser(parser);
+ }
}
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/iwork/IWorkPackageParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/iwork/IWorkPackageParser.java
index 79d82e8..17d30e9 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/iwork/IWorkPackageParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/iwork/IWorkPackageParser.java
@@ -25,6 +25,7 @@ import java.util.HashSet;
import java.util.Set;
import javax.xml.namespace.QName;
+import javax.xml.parsers.SAXParser;
import org.apache.commons.compress.archivers.zip.UnsupportedZipFeatureException;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
@@ -203,10 +204,17 @@ public class IWorkPackageParser extends AbstractParser {
metadata.add(Metadata.CONTENT_TYPE, type.getType().toString());
xhtml.startDocument();
if (contentHandler != null) {
- context.getSAXParser().parse(
- new CloseShieldInputStream(entryStream),
- new OfflineContentHandler(contentHandler)
- );
+ SAXParser parser = null;
+ try {
+ parser = context.acquireSAXParser();
+
+ parser.parse(
+ new CloseShieldInputStream(entryStream),
+ new OfflineContentHandler(contentHandler)
+ );
+ } finally {
+ context.releaseParser(parser);
+ }
}
xhtml.endDocument();
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
index 288e99c..2453cb9 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
@@ -68,6 +68,8 @@ import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;
+import javax.xml.parsers.SAXParser;
+
/**
* Base class for all Tika OOXML extractors.
* <p/>
@@ -515,13 +517,17 @@ public abstract class AbstractOOXMLExtractor implements OOXMLExtractor {
PackageRelationship relatedPartPackageRelationship = relatedPartPRC.getRelationship(i);
try {
PackagePart relatedPartPart = parentPart.getRelatedPart(relatedPartPackageRelationship);
+ SAXParser parser = null;
try (InputStream stream = relatedPartPart.getInputStream()) {
- context.getSAXParser().parse(stream,
+ parser = context.acquireSAXParser();
+ parser.parse(stream,
new OfflineContentHandler(new EmbeddedContentHandler(contentHandler)));
} catch (IOException|TikaException e) {
parentMetadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
ExceptionUtils.getStackTrace(e));
+ } finally {
+ context.releaseParser(parser);
}
} catch (InvalidFormatException e) {
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXSLFPowerPointExtractorDecorator.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXSLFPowerPointExtractorDecorator.java
index 47d0a07..46cfc98 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXSLFPowerPointExtractorDecorator.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXSLFPowerPointExtractorDecorator.java
@@ -49,6 +49,8 @@ import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
+import javax.xml.parsers.SAXParser;
+
/**
* SAX/Streaming pptx extractior
*/
@@ -158,14 +160,18 @@ public class SXSLFPowerPointExtractorDecorator extends AbstractOOXMLExtractor {
if (commentAuthorsPart == null) {
continue;
}
+ SAXParser parser = null;
try (InputStream stream = commentAuthorsPart.getInputStream()) {
- context.getSAXParser().parse(
+ parser = context.acquireSAXParser();
+ parser.parse(
new CloseShieldInputStream(stream),
new OfflineContentHandler(new XSLFCommentAuthorHandler()));
} catch (TikaException | SAXException | IOException e) {
metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
ExceptionUtils.getStackTrace(e));
+ } finally {
+ context.releaseParser(parser);
}
}
@@ -176,8 +182,10 @@ public class SXSLFPowerPointExtractorDecorator extends AbstractOOXMLExtractor {
// Map<String, String> hyperlinks = loadHyperlinkRelationships(packagePart);
xhtml.startElement("div", "class", "slide-content");
+ SAXParser parser = null;
try (InputStream stream = slidePart.getInputStream()) {
- context.getSAXParser().parse(
+ parser = context.acquireSAXParser();
+ parser.parse(
new CloseShieldInputStream(stream),
new OfflineContentHandler(new EmbeddedContentHandler(
new OOXMLWordAndPowerPointTextHandler(
@@ -186,6 +194,8 @@ public class SXSLFPowerPointExtractorDecorator extends AbstractOOXMLExtractor {
} catch (TikaException e) {
metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
ExceptionUtils.getStackTrace(e));
+ } finally {
+ context.releaseParser(parser);
}
xhtml.endElement("div");
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java
index 287a129..82919d3 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java
@@ -47,6 +47,8 @@ import org.apache.tika.utils.ExceptionUtils;
import org.apache.xmlbeans.XmlException;
import org.xml.sax.SAXException;
+import javax.xml.parsers.SAXParser;
+
/**
* This is an experimental, alternative extractor for docx files.
* This streams the main document content rather than loading the
@@ -187,8 +189,10 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor {
XWPFListManager listManager, XHTMLContentHandler xhtml) throws IOException, SAXException {
Map<String, String> linkedRelationships = loadLinkedRelationships(packagePart, true, metadata);
+ SAXParser parser = null;
try (InputStream stream = packagePart.getInputStream()) {
- context.getSAXParser().parse(
+ parser = context.acquireSAXParser();
+ parser.parse(
new CloseShieldInputStream(stream),
new OfflineContentHandler(new EmbeddedContentHandler(
new OOXMLWordAndPowerPointTextHandler(
@@ -197,7 +201,8 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor {
} catch (TikaException e) {
metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
ExceptionUtils.getStackTrace(e));
-
+ } finally {
+ context.releaseParser(parser);
}
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java
index c3b7285..06f8cbf 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java
@@ -195,11 +195,14 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
}
//consider adding this back to POI
+ SAXParser parser = null;
try (InputStream wbData = xssfReader.getWorkbookData()) {
- SAXParser parser = parseContext.getSAXParser();
+ parser = parseContext.acquireSAXParser();
parser.parse(wbData, new OfflineContentHandler(new AbsPathExtractorHandler()));
} catch (InvalidFormatException|TikaException e) {
//swallow
+ } finally {
+ parseContext.releaseParser(parser);
}
}
@@ -335,8 +338,11 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
InputStream sheetInputStream)
throws IOException, SAXException {
InputSource sheetSource = new InputSource(sheetInputStream);
+ SAXParser parser = null;
try {
- XMLReader sheetParser = parseContext.getXMLReader();
+ parser = parseContext.acquireSAXParser();
+
+ XMLReader sheetParser = parser.getXMLReader();
XSSFSheetInterestingPartsCapturer handler =
new XSSFSheetInterestingPartsCapturer(new XSSFSheetXMLHandler(
styles, comments, strings, sheetContentsExtractor, formatter, false));
@@ -349,6 +355,8 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
}
} catch (TikaException e) {
throw new RuntimeException("SAX parser appears to be broken - " + e.getMessage());
+ } finally {
+ parseContext.releaseParser(parser);
}
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xps/XPSExtractorDecorator.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xps/XPSExtractorDecorator.java
index 689db28..9ef992d 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xps/XPSExtractorDecorator.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xps/XPSExtractorDecorator.java
@@ -40,6 +40,7 @@ import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
+import javax.xml.parsers.SAXParser;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
@@ -128,12 +129,15 @@ public class XPSExtractorDecorator extends AbstractOOXMLExtractor {
private void handleDocuments(PackageRelationship packageRelationship,
XHTMLContentHandler xhtml) throws IOException, SAXException, TikaException {
-
+ SAXParser parser = null;
try (InputStream stream = pkg.getPart(packageRelationship).getInputStream()) {
- context.getSAXParser().parse(
+ parser = context.acquireSAXParser();
+ parser.parse(
new CloseShieldInputStream(stream),
new OfflineContentHandler(new EmbeddedContentHandler(
new FixedDocSeqHandler(xhtml))));
+ } finally {
+ context.releaseParser(parser);
}
}
@@ -179,14 +183,18 @@ public class XPSExtractorDecorator extends AbstractOOXMLExtractor {
}
String zipPath = (docRef.startsWith("/") ? docRef.substring(1) : docRef);
if (pkg instanceof ZipPackage) {
+ SAXParser parser = null;
try (InputStream stream = getZipStream(zipPath, pkg)) {
- context.getSAXParser().parse(
+ parser = context.acquireSAXParser();
+ parser.parse(
new CloseShieldInputStream(stream),
new OfflineContentHandler(new EmbeddedContentHandler(
new PageContentPartHandler(relativeRoot, xhtml))));
} catch (IOException | TikaException e) {
throw new SAXException(new TikaException("IOException trying to read: " + docRef));
+ } finally {
+ context.releaseParser(parser);
}
} else {
throw new SAXException(new TikaException("Package must be ZipPackage"));
@@ -226,8 +234,10 @@ public class XPSExtractorDecorator extends AbstractOOXMLExtractor {
if (pagePath.startsWith("/")) {
pagePath = pagePath.substring(1);
}
+ SAXParser parser = null;
try (InputStream stream = getZipStream(pagePath, pkg)) {
- context.getSAXParser().parse(
+ parser = context.acquireSAXParser();
+ parser.parse(
new CloseShieldInputStream(stream),
new OfflineContentHandler(
new XPSPageContentHandler(xhtml, embeddedImages)
@@ -235,6 +245,8 @@ public class XPSExtractorDecorator extends AbstractOOXMLExtractor {
);
} catch (TikaException | IOException e) {
throw new SAXException(e);
+ } finally {
+ context.releaseParser(parser);
}
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFStylesShim.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFStylesShim.java
index 0883747..1b5e76b 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFStylesShim.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFStylesShim.java
@@ -31,6 +31,8 @@ import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
+import javax.xml.parsers.SAXParser;
+
/**
* For Tika, all we need (so far) is a mapping between styleId and a style's name.
*
@@ -58,8 +60,14 @@ public class XWPFStylesShim {
}
private void onDocumentLoad(ParseContext parseContext, InputStream stream) throws TikaException, IOException, SAXException {
- parseContext.getSAXParser().parse(stream,
- new OfflineContentHandler(new StylesStripper()));
+ SAXParser parser = null;
+ try {
+ parser = parseContext.acquireSAXParser();
+ parser.parse(stream,
+ new OfflineContentHandler(new StylesStripper()));
+ } finally {
+ parseContext.releaseParser(parser);
+ }
}
/**
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParser.java
index ff8a43d..4b78e6f 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParser.java
@@ -34,6 +34,8 @@ import org.apache.tika.sax.XHTMLContentHandler;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
+import javax.xml.parsers.SAXParser;
+
public class Word2006MLParser extends AbstractOfficeParser {
@@ -55,15 +57,17 @@ public class Word2006MLParser extends AbstractOfficeParser {
new XHTMLContentHandler(handler, metadata);
xhtml.startDocument();
-
+ SAXParser parser = null;
try {
- context.getSAXParser().parse(
+ parser = context.acquireSAXParser();
+ parser.parse(
new CloseShieldInputStream(stream),
new OfflineContentHandler(new EmbeddedContentHandler(
new Word2006MLDocHandler(xhtml, metadata, context))));
} catch (SAXException e) {
throw new TikaException("XML parse error", e);
} finally {
+ context.releaseParser(parser);
xhtml.endDocument();
}
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/xml/AbstractXML2003Parser.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/xml/AbstractXML2003Parser.java
index 18e5486..8ad4fe2 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/xml/AbstractXML2003Parser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/xml/AbstractXML2003Parser.java
@@ -40,6 +40,8 @@ import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;
+import javax.xml.parsers.SAXParser;
+
public abstract class AbstractXML2003Parser extends AbstractParser {
@@ -94,8 +96,10 @@ public abstract class AbstractXML2003Parser extends AbstractParser {
xhtml.startDocument();
TaggedContentHandler tagged = new TaggedContentHandler(xhtml);
+ SAXParser parser = null;
try {
- context.getSAXParser().parse(
+ parser = context.acquireSAXParser();
+ parser.parse(
new CloseShieldInputStream(stream),
new OfflineContentHandler(new EmbeddedContentHandler(
getContentHandler(tagged, metadata, context))));
@@ -103,6 +107,7 @@ public abstract class AbstractXML2003Parser extends AbstractParser {
tagged.throwIfCauseOf(e);
throw new TikaException("XML parse error", e);
} finally {
+ context.releaseParser(parser);
xhtml.endDocument();
}
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
index 478fed7..7d53976 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
@@ -592,10 +592,17 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
if (parseContext == null) {
parseContext = new ParseContext();
}
- SAXParser parser = parseContext.getSAXParser();
+
xhtml.startElement("div", "class", "ocr");
- parser.parse(is, new OfflineContentHandler(new HOCRPassThroughHandler(xhtml)));
+ SAXParser parser = null;
+ try {
+ parser = parseContext.acquireSAXParser();
+ parser.parse(is, new OfflineContentHandler(new HOCRPassThroughHandler(xhtml)));
+ } finally {
+ parseContext.releaseParser(parser);
+ }
xhtml.endElement("div");
+
}
/**
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
index 18ebc49..da71755 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
@@ -596,11 +596,15 @@ public class OpenDocumentContentParser extends AbstractParser {
DefaultHandler dh = new OpenDocumentElementMappingContentHandler(handler, MAPPINGS);
- SAXParser parser = context.getSAXParser();
- parser.parse(
- new CloseShieldInputStream(stream),
- new OfflineContentHandler(
- new NSNormalizerContentHandler(dh)));
+ SAXParser parser = context.acquireSAXParser();
+ try {
+ parser.parse(
+ new CloseShieldInputStream(stream),
+ new OfflineContentHandler(
+ new NSNormalizerContentHandler(dh)));
+ } finally {
+ context.releaseParser(parser);
+ }
}
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/xml/XMLParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/xml/XMLParser.java
index b17058d..328b711 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/xml/XMLParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/xml/XMLParser.java
@@ -37,6 +37,8 @@ import org.apache.tika.sax.XHTMLContentHandler;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
+import javax.xml.parsers.SAXParser;
+
/**
* XML parser.
*/
@@ -68,8 +70,10 @@ public class XMLParser extends AbstractParser {
xhtml.startElement("p");
TaggedContentHandler tagged = new TaggedContentHandler(handler);
+ SAXParser parser = null;
try {
- context.getSAXParser().parse(
+ parser = context.acquireSAXParser();
+ parser.parse(
new CloseShieldInputStream(stream),
new OfflineContentHandler(new EmbeddedContentHandler(
getContentHandler(tagged, metadata, context))));
@@ -77,6 +81,7 @@ public class XMLParser extends AbstractParser {
tagged.throwIfCauseOf(e);
throw new TikaException("XML parse error", e);
} finally {
+ context.releaseParser(parser);
xhtml.endElement("p");
xhtml.endDocument();
}
diff --git a/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java b/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java
index 6828342..f7644e8 100644
--- a/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java
+++ b/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java
@@ -31,12 +31,14 @@ import org.apache.poi.poifs.filesystem.NPOIFSFileSystem;
import org.apache.tika.MultiThreadedTikaTest;
import org.apache.tika.Tika;
import org.apache.tika.config.TikaConfig;
+import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
import org.apache.tika.mime.MimeTypes;
import org.apache.tika.parser.iwork.iwana.IWork13PackageParser;
import org.apache.tika.utils.XMLReaderUtils;
+import org.junit.After;
import org.junit.Test;
/**
@@ -47,6 +49,12 @@ public class TestContainerAwareDetector extends MultiThreadedTikaTest {
private final MimeTypes mimeTypes = tikaConfig.getMimeRepository();
private final Detector detector = new DefaultDetector(mimeTypes);
+ @After
+ public void tearDown() throws TikaException {
+ //make sure to reset pool size because it is being randomly resized during the tests
+ XMLReaderUtils.setPoolSize(10);
+ }
+
private void assertTypeByData(String file, String type) throws Exception {
assertTypeByNameAndData(file, null, type);
}
@@ -454,7 +462,7 @@ public class TestContainerAwareDetector extends MultiThreadedTikaTest {
};
int numThreads = 1;
XMLReaderUtils.setPoolSize(numThreads);
- testDetector(detector, numThreads, 20, filter, numThreads*2);
+ testDetector(detector, numThreads, 20, filter, numThreads * 2);
}
@Test
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.
[tika] 03/05: TIKA-2645 - use a pool for SAXParsers -- improve
comments and avoid permanent hangs if a parser has forgotten to release its
SAXParser.
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit aa1a749af66e5223665782c3184adc2c1152952f
Author: tballison <ta...@mitre.org>
AuthorDate: Mon May 21 14:08:16 2018 -0400
TIKA-2645 - use a pool for SAXParsers -- improve comments and avoid
permanent hangs if a parser has forgotten to release its SAXParser.
---
tika-core/src/main/java/org/apache/tika/parser/ParseContext.java | 5 +++--
tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java | 8 ++++++--
.../src/main/java/org/apache/tika/parser/dif/DIFParser.java | 2 ++
.../tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParser.java | 2 ++
.../apache/tika/parser/microsoft/xml/AbstractXML2003Parser.java | 2 ++
.../src/main/java/org/apache/tika/parser/xml/XMLParser.java | 2 ++
6 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java b/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
index 6ad85ff..45d0c8f 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
@@ -141,8 +141,9 @@ public class ParseContext implements Serializable {
* Returns the SAX parser specified in this parsing context. If a parser
* is not explicitly specified, then one is acquired from the pool.
* <p>
- * Make sure to {@link #releaseParser(SAXParser)} in
- * a <code>finally</code> block every time you call this.
+ * Make sure to {@link #releaseParser(SAXParser)} as the
+ * first call in a <code>finally</code> block every time
+ * you call this!
* </p>
*
* @return SAXParser
diff --git a/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java b/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java
index a2ad9fc..c48a078 100644
--- a/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java
+++ b/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java
@@ -304,9 +304,13 @@ public class XMLReaderUtils implements Serializable {
}
waiting++;
if (waiting > 3000) {
+ //freshen the pool. Something went very wrong...
+ setPoolSize(POOL_SIZE);
//better to get an exception than have permahang by a bug in one of our parsers
- throw new TikaException("Waited more than 5 minutes for a SAXParser; this could indicate SAXParser leakage. " +
- "Please report this to the Tika team dev@tika.apache.org");
+ throw new TikaException("Waited more than 5 minutes for a SAXParser; " +
+ "This could indicate that a parser has not correctly released its SAXParser. " +
+ "Please report this to the Tika team: dev@tika.apache.org");
+
}
}
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/dif/DIFParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/dif/DIFParser.java
index 237fa0b..33dad3c 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/dif/DIFParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/dif/DIFParser.java
@@ -74,6 +74,8 @@ public class DIFParser extends AbstractParser {
tagged.throwIfCauseOf(e);
throw new TikaException("XML parse error", e);
} finally {
+ //make sure to release the parser before doing
+ //something that could cause an exception!
context.releaseParser(parser);
xhtml.endElement("p");
xhtml.endDocument();
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParser.java
index 4b78e6f..c517ba1 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/ml2006/Word2006MLParser.java
@@ -67,6 +67,8 @@ public class Word2006MLParser extends AbstractOfficeParser {
} catch (SAXException e) {
throw new TikaException("XML parse error", e);
} finally {
+ //make sure to release the parser before doing
+ //something that could cause an exception!
context.releaseParser(parser);
xhtml.endDocument();
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/xml/AbstractXML2003Parser.java b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/xml/AbstractXML2003Parser.java
index 8ad4fe2..226c7b7 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/xml/AbstractXML2003Parser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/xml/AbstractXML2003Parser.java
@@ -107,6 +107,8 @@ public abstract class AbstractXML2003Parser extends AbstractParser {
tagged.throwIfCauseOf(e);
throw new TikaException("XML parse error", e);
} finally {
+ //make sure to release the parser before doing
+ //something that could cause an exception!
context.releaseParser(parser);
xhtml.endDocument();
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/xml/XMLParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/xml/XMLParser.java
index 328b711..f05e343 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/xml/XMLParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/xml/XMLParser.java
@@ -81,6 +81,8 @@ public class XMLParser extends AbstractParser {
tagged.throwIfCauseOf(e);
throw new TikaException("XML parse error", e);
} finally {
+ //make sure to release the parser before doing
+ //something that could cause an exception!
context.releaseParser(parser);
xhtml.endElement("p");
xhtml.endDocument();
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.
[tika] 04/05: TIKA-2645 - remove commons math from dependencies
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 017096f64b5fba27f4208a65d636ee53b22f520f
Author: tballison <ta...@mitre.org>
AuthorDate: Mon May 21 14:13:57 2018 -0400
TIKA-2645 - remove commons math from dependencies
---
tika-parsers/pom.xml | 6 ------
1 file changed, 6 deletions(-)
diff --git a/tika-parsers/pom.xml b/tika-parsers/pom.xml
index fc1a028..779d660 100644
--- a/tika-parsers/pom.xml
+++ b/tika-parsers/pom.xml
@@ -732,12 +732,6 @@
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- <version>3.6.1</version>
- <scope>test</scope>
- </dependency>
</dependencies>
<build>
<plugins>
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.