You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2021/01/13 19:39:54 UTC
[tika] 04/04: TIKA-3258 -- in Tika 2.0.0,
the default for OCR'ing of PDFs is 'auto'
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 4bd897a7df772c208fb6918b6b7559e37e5ec3b9
Author: tallison <ta...@apache.org>
AuthorDate: Wed Jan 13 14:39:27 2021 -0500
TIKA-3258 -- in Tika 2.0.0, the default for OCR'ing of PDFs is 'auto'
---
CHANGES.txt | 3 +
.../apache/tika/parser/ocr/TesseractOCRParser.java | 36 +++------
.../apache/tika/parser/pdf/AbstractPDF2XHTML.java | 26 ++++---
.../java/org/apache/tika/parser/pdf/OCR2XHTML.java | 2 +-
.../apache/tika/parser/pdf/PDFParserConfig.java | 2 +-
.../apache/tika/parser/pdf/PDFParser.properties | 2 +-
.../org/apache/tika/parser/pdf/PDFParserTest.java | 88 ++++++++++++++--------
7 files changed, 89 insertions(+), 70 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 6a9d1a6..0a12cb1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -2,6 +2,9 @@ Release 2.0.0-ALPHA - 01/13/2021
BREAKING CHANGES in 2.0.0
* General
+ * OCR is now triggered automatically for PDFs if tesseract
+ is on the user's path see (https://cwiki.apache.org/confluence/display/TIKA/TikaOCR#TikaOCR-disable-ocr)
+ for how to disable OCR.
* Remove deprecated Metadata keys/properties (TIKA-1974).
* Removed dangerous calls to read an inputstream or convert to bytes
without specifying a charset
diff --git a/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java b/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
index 28fb5b8..264295c 100644
--- a/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
+++ b/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
@@ -88,7 +88,7 @@ import static org.apache.tika.sax.XHTMLContentHandler.XHTML;
*
*
*/
-public class TesseractOCRParser extends AbstractParser implements Initializable {
+public class TesseractOCRParser extends AbstractParser {
public static final String TESS_META = "tess:";
public static final Property IMAGE_ROTATION = Property.externalRealSeq(TESS_META+"rotation");
public static final Property IMAGE_MAGICK = Property.externalBooleanSeq(TESS_META+"image_magick_processed");
@@ -230,6 +230,7 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
ContentHandler xhtml, Metadata metadata, ParseContext parseContext,
TesseractOCRConfig config)
throws IOException, SAXException, TikaException {
+ warnOnFirstParse();
File tmpTxtOutput = null;
try {
Path input = tikaInputStream.getPath();
@@ -284,32 +285,9 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
}
}
- /**
- * no-op
- * @param params params to use for initialization
- * @throws TikaConfigException
- */
- @Override
- public void initialize(Map<String, Param> params) throws TikaConfigException {
-
- }
-
- @Override
- public void checkInitialization(InitializableProblemHandler problemHandler)
- throws TikaConfigException {
- //this will incorrectly trigger for people who turn off Tesseract
- //by sending in a bogus tesseract path via a custom TesseractOCRConfig.
- //TODO: figure out how to solve that.
- if (! hasWarned()) {
- if (hasTesseract(defaultConfig)) {
- problemHandler.handleInitializableProblem(this.getClass().getName(),
- "Tesseract OCR is installed and will be automatically applied to image files unless\n" +
- "you've excluded the TesseractOCRParser from the default parser.\n"+
- "Tesseract may dramatically slow down content extraction (TIKA-2359).\n" +
- "As of Tika 1.15 (and prior versions), Tesseract is automatically called.\n" +
- "In future versions of Tika, users may need to turn the TesseractOCRParser on via TikaConfig.");
- warn();
- }
+ private void warnOnFirstParse() {
+ if (!hasWarned()) {
+ warn();
}
}
@@ -515,6 +493,10 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
}
protected void warn() {
+ LOG.info("Tesseract is installed and is being invoked. " +
+ "This can add greatly to processing time. If you do not want tesseract " +
+ "to be applied to your files see: " +
+ "https://cwiki.apache.org/confluence/display/TIKA/TikaOCR#TikaOCR-disable-ocr");
HAS_WARNED = true;
}
diff --git a/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java b/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
index f60e53c..90ef86b 100644
--- a/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
+++ b/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
@@ -16,7 +16,10 @@
*/
package org.apache.tika.parser.pdf;
+import static org.apache.tika.parser.pdf.PDFParserConfig.OCR_STRATEGY.AUTO;
import static org.apache.tika.parser.pdf.PDFParserConfig.OCR_STRATEGY.NO_OCR;
+import static org.apache.tika.parser.pdf.PDFParserConfig.OCR_STRATEGY.OCR_AND_TEXT_EXTRACTION;
+import static org.apache.tika.parser.pdf.PDFParserConfig.OCR_STRATEGY.OCR_ONLY;
import javax.xml.stream.XMLStreamException;
import java.awt.image.BufferedImage;
@@ -422,18 +425,23 @@ class AbstractPDF2XHTML extends PDFTextStripper {
}
}
- void doOCROnCurrentPage() throws IOException, TikaException, SAXException {
- if (config.getOcrStrategy().equals(NO_OCR)) {
+ void doOCROnCurrentPage(PDFParserConfig.OCR_STRATEGY ocrStrategy) throws IOException, TikaException, SAXException {
+ if (ocrStrategy.equals(NO_OCR)) {
return;
}
MediaType ocrImageMediaType =
MediaType.image("ocr-"+config.getOcrImageFormatName());
if (! ocrParser.getSupportedTypes(context).contains(ocrImageMediaType)) {
- throw new TikaException("" +
- "I regret that I couldn't find an OCR parser to handle "+ocrImageMediaType+"."+
- "Please set the OCR_STRATEGY to NO_OCR or configure your" +
- "OCR parser correctly"
- );
+ if (ocrStrategy == OCR_ONLY || ocrStrategy == OCR_AND_TEXT_EXTRACTION) {
+ throw new TikaException("" +
+ "I regret that I couldn't find an OCR parser to handle " + ocrImageMediaType + "." +
+ "Please set the OCR_STRATEGY to NO_OCR or configure your" +
+ "OCR parser correctly"
+ );
+ } else if (ocrStrategy == AUTO) {
+ //silently skip
+ return;
+ }
}
PDFRenderer renderer = new PDFRenderer(pdDocument);
@@ -544,11 +552,11 @@ class AbstractPDF2XHTML extends PDFTextStripper {
}
}
if (config.getOcrStrategy().equals(PDFParserConfig.OCR_STRATEGY.OCR_AND_TEXT_EXTRACTION)) {
- doOCROnCurrentPage();
+ doOCROnCurrentPage(OCR_AND_TEXT_EXTRACTION);
} else if (config.getOcrStrategy().equals(PDFParserConfig.OCR_STRATEGY.AUTO)) {
//TODO add more sophistication
if (totalCharsPerPage < 10 || unmappedUnicodeCharsPerPage > 10) {
- doOCROnCurrentPage();
+ doOCROnCurrentPage(AUTO);
}
}
diff --git a/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/OCR2XHTML.java b/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/OCR2XHTML.java
index 539cd50..4eb849e 100644
--- a/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/OCR2XHTML.java
+++ b/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/OCR2XHTML.java
@@ -94,7 +94,7 @@ class OCR2XHTML extends AbstractPDF2XHTML {
public void processPage(PDPage pdPage) throws IOException {
try {
startPage(pdPage);
- doOCROnCurrentPage();
+ doOCROnCurrentPage(PDFParserConfig.OCR_STRATEGY.OCR_ONLY);
endPage(pdPage);
} catch (TikaException|SAXException e) {
throw new IOExceptionWithCause(e);
diff --git a/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java b/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
index b031a72..d52f91a 100644
--- a/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
+++ b/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
@@ -131,7 +131,7 @@ public class PDFParserConfig implements Serializable {
//content from elsewhere in the document.
private boolean ifXFAExtractOnlyXFA = false;
- private OCR_STRATEGY ocrStrategy = OCR_STRATEGY.NO_OCR;
+ private OCR_STRATEGY ocrStrategy = OCR_STRATEGY.AUTO;
private int ocrDPI = 300;
private ImageType ocrImageType = ImageType.GRAY;
diff --git a/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-pdf-module/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties b/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-pdf-module/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties
index 28bbbac..e2c0c32 100644
--- a/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-pdf-module/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties
+++ b/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-pdf-module/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties
@@ -26,7 +26,7 @@ allowExtractionForAccessibility true
ifXFAExtractOnlyXFA false
catchIntermediateIOExceptions true
#options: no_ocr, ocr_only, ocr_and_text_extraction, auto
-ocrStrategy no_ocr
+ocrStrategy auto
#dots per inch for the ocr rendering of the page image
ocrDPI 300
#if you request tif, make sure you have imageio jars on your classpath!
diff --git a/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java b/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
index b094374..439a6ea 100644
--- a/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
+++ b/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
@@ -82,11 +82,19 @@ public class PDFParserTest extends TikaTest {
public static final MediaType TYPE_DOCX = MediaType.application("vnd.openxmlformats-officedocument.wordprocessingml.document");
public static final MediaType TYPE_DOC = MediaType.application("msword");
+ private static ParseContext NO_OCR() {
+ PDFParserConfig config = new PDFParserConfig();
+ config.setOcrStrategy(PDFParserConfig.OCR_STRATEGY.NO_OCR);
+ ParseContext context = new ParseContext();
+ context.set(PDFParserConfig.class, config);
+ return context;
+ }
@Test
public void testXMLProfiler() throws Exception {
//test that the xml profiler is not triggered by default
- List<Metadata> metadataList = getRecursiveMetadata("testPDF_XFA_govdocs1_258578.pdf");
+ List<Metadata> metadataList = getRecursiveMetadata("testPDF_XFA_govdocs1_258578.pdf",
+ NO_OCR());
assertEquals(1, metadataList.size());
//test that it is triggered when added to the default parser
@@ -133,7 +141,8 @@ public class PDFParserTest extends TikaTest {
@Test //TIKA-1374
public void testOSSpecificEmbeddedFileExtraction() throws Exception {
- List<Metadata> metadatas = getRecursiveMetadata("testPDF_multiFormatEmbFiles.pdf");
+ List<Metadata> metadatas = getRecursiveMetadata("testPDF_multiFormatEmbFiles.pdf",
+ NO_OCR());
assertEquals("metadata size", 5, metadatas.size());
assertEquals("file name", "Test.txt", metadatas.get(1).get(TikaCoreProperties.RESOURCE_NAME_KEY));
@@ -156,7 +165,7 @@ public class PDFParserTest extends TikaTest {
docx
*/
- String content = getXML("testPDFEmbeddingAndEmbedded.docx").xml;
+ String content = getXML("testPDFEmbeddingAndEmbedded.docx", NO_OCR()).xml;
int outerHaystack = content.indexOf("Outer_haystack");
int pdfHaystack = content.indexOf("pdf_haystack");
int needle = content.indexOf("Needle");
@@ -194,6 +203,7 @@ public class PDFParserTest extends TikaTest {
PDFParserConfig config = new PDFParserConfig();
config.setExtractInlineImages(true);
config.setExtractUniqueInlineImagesOnly(false);
+ config.setOcrStrategy(PDFParserConfig.OCR_STRATEGY.NO_OCR);
context.set(org.apache.tika.parser.pdf.PDFParserConfig.class, config);
context.set(org.apache.tika.parser.Parser.class, p);
@@ -219,53 +229,63 @@ public class PDFParserTest extends TikaTest {
@Test
public void testEmbeddedDocsWithOCROnly() throws Exception {
assumeTrue("can run OCR", canRunOCR());
-
+ //test default is "auto"
+ assertEquals(PDFParserConfig.OCR_STRATEGY.AUTO, new PDFParserConfig().getOcrStrategy());
+ testStrategy(null);
+ //now test other options
for (PDFParserConfig.OCR_STRATEGY strategy : PDFParserConfig.OCR_STRATEGY.values()) {
+ testStrategy(strategy);
+ }
+ }
+
+ private void testStrategy(PDFParserConfig.OCR_STRATEGY strategy) throws Exception {
+ //make sure everything works with regular xml _and_ with recursive
+ ParseContext context = new ParseContext();
+ if (strategy != null) {
PDFParserConfig config = new PDFParserConfig();
config.setOcrStrategy(strategy);
- ParseContext context = new ParseContext();
context.set(PDFParserConfig.class, config);
- //make sure everything works with regular xml _and_ with recursive
- XMLResult xmlResult = getXML("testPDFEmbeddingAndEmbedded.docx", context);
- //can get dehaystack depending on version of tesseract and/or preprocessing
- if (xmlResult.xml.contains("pdf_haystack") || xmlResult.xml.contains("dehaystack")) {
- //great
- } else {
- fail("couldn't find pdf_haystack or its variants");
- }
- assertContains("Haystack", xmlResult.xml);
- assertContains("Needle", xmlResult.xml);
- if (! strategy.equals(PDFParserConfig.OCR_STRATEGY.NO_OCR)) {
- // Tesseract may see the t in haystack as a ! some times...
- //or it might see dehayslack...
- //TODO: figure out how to make this test less hacky
- String div = "<div class=\"ocr\">";
- if (xmlResult.xml.contains(div+"pdf_hays!ack")) {
- } else if (xmlResult.xml.contains(div+"pdf_haystack")) {
- } else if (xmlResult.xml.contains(div+"dehayslack")) {
- } else {
- fail("couldn't find acceptable variants of haystack");
- }
+ }
+ XMLResult xmlResult = getXML("testPDFEmbeddingAndEmbedded.docx", context);
+
+ //can get dehaystack depending on version of tesseract and/or preprocessing
+ if (xmlResult.xml.contains("pdf_haystack") || xmlResult.xml.contains("dehaystack")) {
+ //great
+ } else {
+ fail("couldn't find pdf_haystack or its variants");
+ }
+ assertContains("Haystack", xmlResult.xml);
+ assertContains("Needle", xmlResult.xml);
+ if (strategy == null || strategy != PDFParserConfig.OCR_STRATEGY.NO_OCR) {
+ // Tesseract may see the t in haystack as a ! some times...
+ //or it might see dehayslack...
+ //TODO: figure out how to make this test less hacky
+ String div = "<div class=\"ocr\">";
+ if (xmlResult.xml.contains(div+"pdf_hays!ack")) {
+ } else if (xmlResult.xml.contains(div+"pdf_haystack")) {
+ } else if (xmlResult.xml.contains(div+"dehayslack")) {
} else {
- assertNotContained("<div class=\"ocr\">pdf_haystack", xmlResult.xml);
+ fail("couldn't find acceptable variants of haystack");
}
- assertEquals(4, getRecursiveMetadata("testPDFEmbeddingAndEmbedded.docx", context).size());
+ } else {
+ assertNotContained("<div class=\"ocr\">pdf_haystack", xmlResult.xml);
}
-
+ assertEquals(4, getRecursiveMetadata("testPDFEmbeddingAndEmbedded.docx", context).size());
}
@Test
public void testFileInAnnotationExtractedIfNoContents() throws Exception {
//TIKA-2845
- List<Metadata> contents = getRecursiveMetadata("testPDFFileEmbInAnnotation_noContents.pdf");
+ List<Metadata> contents = getRecursiveMetadata("testPDFFileEmbInAnnotation_noContents.pdf",
+ NO_OCR());
assertEquals(2, contents.size());
assertContains("This is a Excel", contents.get(1).get(RecursiveParserWrapperHandler.TIKA_CONTENT));
}
@Test
public void testEmbeddedFilesInAnnotations() throws Exception {
- String xml = getXML("testPDFFileEmbInAnnotation.pdf").xml;
+ String xml = getXML("testPDFFileEmbInAnnotation.pdf", NO_OCR()).xml;
assertTrue(xml.contains("This is a Excel"));
}
@@ -275,6 +295,7 @@ public class PDFParserTest extends TikaTest {
//TIKA-1990, test that an embedded jpeg is correctly decoded
PDFParserConfig config = new PDFParserConfig();
config.setExtractInlineImages(true);
+ config.setOcrStrategy(PDFParserConfig.OCR_STRATEGY.NO_OCR);
ParseContext context = new ParseContext();
context.set(PDFParserConfig.class, config);
@@ -295,6 +316,7 @@ public class PDFParserTest extends TikaTest {
PDFParserConfig config = new PDFParserConfig();
config.setExtractInlineImages(true);
config.setExtractUniqueInlineImagesOnly(false);
+ config.setOcrStrategy(PDFParserConfig.OCR_STRATEGY.NO_OCR);
context.set(PDFParserConfig.class, config);
@@ -332,6 +354,10 @@ public class PDFParserTest extends TikaTest {
@Test
public void testOCRAutoMode() throws Exception {
assumeTrue("can run OCR", canRunOCR());
+
+ //default
+ assertContains("Happy New Year", getXML("testOCR.pdf").xml);
+
PDFParserConfig config = new PDFParserConfig();
config.setOcrStrategy(PDFParserConfig.OCR_STRATEGY.AUTO);
ParseContext context = new ParseContext();