You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2016/04/13 02:59:47 UTC
tika git commit: TIKA-1948 handle IOExceptions per page in PDFParser
Repository: tika
Updated Branches:
refs/heads/2.x c93412554 -> 2e2b96a1b
TIKA-1948 handle IOExceptions per page in PDFParser
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/2e2b96a1
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/2e2b96a1
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/2e2b96a1
Branch: refs/heads/2.x
Commit: 2e2b96a1b940d6e4de649931b337ab290e8504e2
Parents: c934125
Author: tballison <ta...@mitre.org>
Authored: Tue Apr 12 20:59:35 2016 -0400
Committer: tballison <ta...@mitre.org>
Committed: Tue Apr 12 20:59:35 2016 -0400
----------------------------------------------------------------------
CHANGES.txt | 4 +-
.../tika/metadata/TikaCoreProperties.java | 9 +++
.../org/apache/tika/parser/pdf/PDF2XHTML.java | 59 ++++++++++++++++---
.../apache/tika/parser/pdf/PDFParserConfig.java | 32 ++++++++++
.../apache/tika/parser/pdf/PDFParser.properties | 3 +-
.../apache/tika/parser/pdf/PDFParserTest.java | 39 ++++++++++++
.../test-documents/testPDF_bad_page_303226.pdf | Bin 0 -> 138027 bytes
7 files changed, 137 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/2e2b96a1/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e446fa6..0a80ef7 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -10,6 +10,8 @@ Release 2.0 - Future Development
Release 1.13 - ???
+ * Handle per page IOExceptions more robustly in PDFParser (TIKA-1948).
+
* Upgrade commons-compress to 1.11 (TIKA-1949).
* Add detection for embedded MSChart.Graph files (TIKA-1033).
@@ -36,7 +38,7 @@ Release 1.13 - ???
https://pdfbox.apache.org/2.0/dependencies.html#optional-components
for optional components to process Tiff files.
* Some truncated/corrupted files that had some content extracted
- with 1.8.11 may have no content extracted in 2.0.0.
+ with 1.8.11 may have no content extracted in 2.0.0 (see TIKA-1912).
* Add XMPMM support to PDFParser and JpegParser via Jempbox (TIKA-1894).
http://git-wip-us.apache.org/repos/asf/tika/blob/2e2b96a1/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java
----------------------------------------------------------------------
diff --git a/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java b/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java
index 6f2ba2a..5052fbc 100644
--- a/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java
+++ b/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java
@@ -67,6 +67,15 @@ public interface TikaCoreProperties {
Metadata.NAMESPACE_PREFIX_DELIMITER;
/**
+ * Use this to store exceptions caught during a parse that are
+ * non-fatal, e.g. if a parser is in lenient mode and more
+ * content can be extracted if we ignore an exception thrown by
+ * a dependency.
+ */
+ public static final Property TIKA_META_EXCEPTION_WARNING =
+ Property.internalTextBag(TIKA_META_EXCEPTION_PREFIX+"warn");
+
+ /**
* This is currently used to identify Content-Type that may be
* included within a document, such as in html documents
* (e.g. <meta http-equiv="content-type" content="text/html; charset=UTF-8">)
http://git-wip-us.apache.org/repos/asf/tika/blob/2e2b96a1/tika-parser-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java b/tika-parser-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
index e442d15..d242ec7 100644
--- a/tika-parser-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
+++ b/tika-parser-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
@@ -26,6 +26,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.Writer;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.Calendar;
import java.util.HashMap;
import java.util.HashSet;
@@ -107,6 +108,7 @@ class PDF2XHTML extends PDFTextStripper {
private final XHTMLContentHandler handler;
private final PDFParserConfig config;
private final Metadata metadata;
+ private final List<IOException> exceptions = new ArrayList<>();
/**
* This keeps track of the pdf object ids for inline
* images that have been processed.
@@ -140,17 +142,18 @@ class PDF2XHTML extends PDFTextStripper {
* @param handler SAX content handler
* @param metadata PDF metadata
* @throws SAXException if the content handler fails to process SAX events
- * @throws TikaException if the PDF document can not be processed
+ * @throws TikaException if there was an exception outside of per page processing
*/
public static void process(
PDDocument document, ContentHandler handler, ParseContext context, Metadata metadata,
PDFParserConfig config)
throws SAXException, TikaException {
+ PDF2XHTML pdf2XHTML = null;
try {
// Extract text using a dummy Writer as we override the
// key methods to output to the given content
// handler.
- PDF2XHTML pdf2XHTML = new PDF2XHTML(handler, context, metadata, config);
+ pdf2XHTML = new PDF2XHTML(handler, context, metadata, config);
config.configure(pdf2XHTML);
@@ -175,6 +178,11 @@ class PDF2XHTML extends PDFTextStripper {
throw new TikaException("Unable to extract PDF content", e);
}
}
+ if (pdf2XHTML.exceptions.size() > 0) {
+ //throw the first
+ throw new TikaException("Unable to extract all PDF content",
+ pdf2XHTML.exceptions.get(0));
+ }
}
void extractBookmarkText() throws SAXException {
@@ -201,6 +209,15 @@ class PDF2XHTML extends PDFTextStripper {
}
@Override
+ public void processPage(PDPage page) throws IOException {
+ try {
+ super.processPage(page);
+ } catch (IOException e) {
+ handleCatchableIOE(e);
+ }
+ }
+
+ @Override
protected void startDocument(PDDocument pdf) throws IOException {
try {
handler.startDocument();
@@ -214,11 +231,19 @@ class PDF2XHTML extends PDFTextStripper {
try {
// Extract text for any bookmarks:
extractBookmarkText();
- extractEmbeddedDocuments(pdf, originalHandler);
+ try {
+ extractEmbeddedDocuments(pdf, originalHandler);
+ } catch (IOException e) {
+ handleCatchableIOE(e);
+ }
//extract acroform data at end of doc
if (config.getExtractAcroFormContent() == true) {
- extractAcroForm(pdf, handler);
+ try {
+ extractAcroForm(pdf, handler);
+ } catch (IOException e) {
+ handleCatchableIOE(e);
+ }
}
handler.endDocument();
} catch (TikaException e) {
@@ -242,8 +267,11 @@ class PDF2XHTML extends PDFTextStripper {
protected void endPage(PDPage page) throws IOException {
try {
writeParagraphEnd();
-
- extractImages(page.getResources(), new HashSet<COSBase>());
+ try {
+ extractImages(page.getResources(), new HashSet<COSBase>());
+ } catch (IOException e) {
+ handleCatchableIOE(e);
+ }
EmbeddedDocumentExtractor extractor = getEmbeddedDocumentExtractor();
for (PDAnnotation annotation : page.getAnnotations()) {
@@ -257,6 +285,8 @@ class PDF2XHTML extends PDFTextStripper {
throw new IOExceptionWithCause("file embedded in annotation sax exception", e);
} catch (TikaException e) {
throw new IOExceptionWithCause("file embedded in annotation tika exception", e);
+ } catch (IOException e) {
+ handleCatchableIOE(e);
}
}
// TODO: remove once PDFBOX-1143 is fixed:
@@ -314,6 +344,8 @@ class PDF2XHTML extends PDFTextStripper {
handler.endElement("div");
} catch (SAXException e) {
throw new IOExceptionWithCause("Unable to end a page", e);
+ } catch (IOException e) {
+ exceptions.add(e);
}
}
@@ -395,7 +427,7 @@ class PDF2XHTML extends PDFTextStripper {
new EmbeddedContentHandler(handler),
metadata, false);
} catch (IOException e) {
- // could not extract this image, so just skip it...
+ handleCatchableIOE(e);
}
}
}
@@ -749,5 +781,18 @@ class PDF2XHTML extends PDFTextStripper {
handler.endElement("li");
}
}
+
+ private void handleCatchableIOE(IOException e) throws IOException {
+ if (config.isCatchIntermediateIOExceptions()) {
+ String msg = e.getMessage();
+ if (msg == null) {
+ msg = "IOException, no message";
+ }
+ metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, msg);
+ exceptions.add(e);
+ } else {
+ throw e;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tika/blob/2e2b96a1/tika-parser-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java b/tika-parser-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
index ea43761..9baeb37 100644
--- a/tika-parser-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
+++ b/tika-parser-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
@@ -82,6 +82,12 @@ public class PDFParserConfig implements Serializable {
private AccessChecker accessChecker;
+ //The PDFParser can throw IOExceptions if there is a problem
+ //with a streams. If this is set to true, Tika's
+ //parser catches these exceptions, reports them in the metadata
+ //and then throws the first stored exception after the parse has completed.
+ private boolean isCatchIntermediateIOExceptions = true;
+
public PDFParserConfig() {
init(this.getClass().getResourceAsStream("PDFParser.properties"));
}
@@ -141,6 +147,10 @@ public class PDFParserConfig implements Serializable {
getProp(props.getProperty("ifXFAExtractOnlyXFA"),
getIfXFAExtractOnlyXFA()));
+ setCatchIntermediateIOExceptions(
+ getProp(props.getProperty("catchIntermediateIOExceptions"),
+ isCatchIntermediateIOExceptions()));
+
boolean checkExtractAccessPermission = getProp(props.getProperty("checkExtractAccessPermission"), false);
boolean allowExtractionForAccessibility = getProp(props.getProperty("allowExtractionForAccessibility"), true);
@@ -378,6 +388,26 @@ public class PDFParserConfig implements Serializable {
this.accessChecker = accessChecker;
}
+ /**
+ * See {@link #setCatchIntermediateIOExceptions(boolean)}
+ * @return whether or not to catch IOExceptions
+ */
+ public boolean isCatchIntermediateIOExceptions() {
+ return isCatchIntermediateIOExceptions;
+ }
+
+ /**
+ * The PDFBox parser will throw an IOException if there is
+ * a problem with a stream. If this is set to <code>true</code>,
+ * Tika's PDFParser will catch these exceptions and try to parse
+ * the rest of the document. After the parse is completed,
+ * Tika's PDFParser will throw the first caught exception.
+ * @param catchIntermediateIOExceptions
+ */
+ public void setCatchIntermediateIOExceptions(boolean catchIntermediateIOExceptions) {
+ isCatchIntermediateIOExceptions = catchIntermediateIOExceptions;
+ }
+
private boolean getProp(String p, boolean defaultMissing) {
if (p == null) {
return defaultMissing;
@@ -467,4 +497,6 @@ public class PDFParserConfig implements Serializable {
+ averageCharTolerance + ", spacingTolerance="
+ spacingTolerance + "]";
}
+
+
}
http://git-wip-us.apache.org/repos/asf/tika/blob/2e2b96a1/tika-parser-modules/tika-parser-pdf-module/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-pdf-module/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties b/tika-parser-modules/tika-parser-pdf-module/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties
index 153950e..9b404a3 100644
--- a/tika-parser-modules/tika-parser-pdf-module/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties
+++ b/tika-parser-modules/tika-parser-pdf-module/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties
@@ -22,4 +22,5 @@ extractInlineImages false
extractUniqueInlineImagesOnly true
checkExtractAccessPermission false
allowExtractionForAccessibility true
-ifXFAExtractOnlyXFA false
\ No newline at end of file
+ifXFAExtractOnlyXFA false
+catchIntermediateIOExceptions true
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/2e2b96a1/tika-parser-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java b/tika-parser-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
index 8794509..bd2e5ad 100644
--- a/tika-parser-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
+++ b/tika-parser-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
@@ -35,6 +35,7 @@ import org.apache.log4j.Logger;
import org.apache.tika.TikaTest;
import org.apache.tika.exception.AccessPermissionException;
import org.apache.tika.exception.EncryptedDocumentException;
+import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.ContainerExtractor;
import org.apache.tika.extractor.DocumentSelector;
import org.apache.tika.extractor.ParserContainerExtractor;
@@ -1119,6 +1120,44 @@ public class PDFParserTest extends TikaTest {
}, m.getValues(XMPMM.HISTORY_WHEN));
}
+ @Test
+ public void testSkipBadPage() throws Exception {
+ //test file comes from govdocs1
+ //can't use TikaTest shortcuts because of exception
+ Parser p = new AutoDetectParser();
+ ContentHandler handler = new BodyContentHandler(-1);
+ Metadata m = new Metadata();
+ ParseContext context = new ParseContext();
+ boolean tikaEx = false;
+ try (InputStream is = getResourceAsStream("/test-documents/testPDF_bad_page_303226.pdf")) {
+ p.parse(is, handler, m, context);
+ } catch (TikaException e) {
+ tikaEx = true;
+ }
+ String content = handler.toString();
+ assertTrue("Should have thrown exception", tikaEx);
+ assertEquals(1, m.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING).length);
+ assertContains("Unknown dir", m.get(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING));
+ assertContains("1309.61", content);
+
+ //now try throwing exception immediately
+ PDFParserConfig config = new PDFParserConfig();
+ config.setCatchIntermediateIOExceptions(false);
+ context.set(PDFParserConfig.class, config);
+
+ handler = new BodyContentHandler(-1);
+ m = new Metadata();
+ tikaEx = false;
+ try (InputStream is = getResourceAsStream("/test-documents/testPDF_bad_page_303226.pdf")) {
+ p.parse(is, handler, m, context);
+ } catch (TikaException e) {
+ tikaEx = true;
+ }
+ content = handler.toString();
+ assertTrue("Should have thrown exception", tikaEx);
+ assertEquals(0, m.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING).length);
+ assertNotContained("1309.61", content);
+ }
private void assertException(String path, Parser parser, ParseContext context, Class expected) {
boolean noEx = false;
InputStream is = getResourceAsStream(path);
http://git-wip-us.apache.org/repos/asf/tika/blob/2e2b96a1/tika-test-resources/src/test/resources/test-documents/testPDF_bad_page_303226.pdf
----------------------------------------------------------------------
diff --git a/tika-test-resources/src/test/resources/test-documents/testPDF_bad_page_303226.pdf b/tika-test-resources/src/test/resources/test-documents/testPDF_bad_page_303226.pdf
new file mode 100644
index 0000000..ba2f9bb
Binary files /dev/null and b/tika-test-resources/src/test/resources/test-documents/testPDF_bad_page_303226.pdf differ