You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2021/10/20 14:02:15 UTC
[tika] branch main updated: TIKA-2696 -- allow tesseract to handle
psm mode 0
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 94f9eb7 TIKA-2696 -- allow tesseract to handle psm mode 0
new 507922c Merge remote-tracking branch 'origin/main' into main
94f9eb7 is described below
commit 94f9eb72e9020c1108d7544135e0b44810dab035
Author: tallison <ta...@apache.org>
AuthorDate: Wed Oct 20 10:01:31 2021 -0400
TIKA-2696 -- allow tesseract to handle psm mode 0
---
.../apache/tika/parser/ocr/TesseractOCRParser.java | 90 ++++++++++++++++++----
.../tika/parser/ocr/TesseractOCRParserTest.java | 21 +++++
.../resources/test-configs/tika-config-psm0.xml | 29 +++++++
3 files changed, 124 insertions(+), 16 deletions(-)
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
index a5ee64f..2cc7aba 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
@@ -21,6 +21,7 @@ import static org.apache.tika.sax.XHTMLContentHandler.XHTML;
import java.awt.Image;
import java.awt.image.BufferedImage;
+import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@@ -42,6 +43,8 @@ import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import javax.imageio.ImageIO;
import org.apache.commons.io.FilenameUtils;
@@ -93,6 +96,17 @@ public class TesseractOCRParser extends AbstractExternalProcessParser implements
Property.externalBooleanSeq(TESS_META + "image_magick_processed");
private static final String TESSDATA_PREFIX = "TESSDATA_PREFIX";
+ public static final Property
+ PSM0_PAGE_NUMBER = Property.externalInteger(TESS_META + "page_number");
+ public static final Property
+ PSM0_ORIENTATION = Property.externalInteger(TESS_META + "orientation");
+ public static final Property PSM0_ROTATE = Property.externalInteger(TESS_META + "rotate");
+ public static final Property PSM0_ORIENTATION_CONFIDENCE = Property.externalReal(TESS_META +
+ "orientation_confidence");
+ public static final Property PSM0_SCRIPT = Property.externalText(TESS_META + "script");
+ public static final Property PSM0_SCRIPT_CONFIDENCE = Property.externalReal(TESS_META +
+ "script_confidence");
+
private static final String OCR = "ocr-";
private static final Logger LOG = LoggerFactory.getLogger(TesseractOCRParser.class);
private static final Object[] LOCK = new Object[0];
@@ -284,13 +298,17 @@ public class TesseractOCRParser extends AbstractExternalProcessParser implements
doOCR(input.toFile(), tmpOCROutputFile, config);
}
- // Tesseract appends the output type (.txt or .hocr) to output file name
- tmpTxtOutput = new File(tmpOCROutputFile.getAbsolutePath() + "." +
- config.getOutputType().toString().toLowerCase(Locale.US));
+ String extension = config.getPageSegMode().equals("0") ? "osd" :
+ config.getOutputType().toString().toLowerCase(Locale.US);
+ // Tesseract appends the output type (.txt or .hocr or .osd) to output file name
+ tmpTxtOutput = new File(tmpOCROutputFile.getAbsolutePath() +
+ "." + extension);
if (tmpTxtOutput.exists()) {
try (InputStream is = new FileInputStream(tmpTxtOutput)) {
- if (config.getOutputType().equals(TesseractOCRConfig.OUTPUT_TYPE.HOCR)) {
+ if (config.getPageSegMode().equals("0")) {
+ extractOSD(is, metadata);
+ } else if (config.getOutputType().equals(TesseractOCRConfig.OUTPUT_TYPE.HOCR)) {
extractHOCROutput(is, parseContext, xhtml);
} else {
extractOutput(is, xhtml);
@@ -305,6 +323,43 @@ public class TesseractOCRParser extends AbstractExternalProcessParser implements
}
}
+ private void extractOSD(InputStream is, Metadata metadata) throws IOException {
+ Matcher matcher = Pattern.compile("^([^:]+):\\s+(.*)").matcher("");
+ try (BufferedReader reader = new BufferedReader(new InputStreamReader(is,
+ UTF_8))) {
+ String line = reader.readLine();
+ while (line != null) {
+ if (matcher.reset(line).find()) {
+ String k = matcher.group(1);
+ String v = matcher.group(2);
+ switch (k) {
+ case "Page number":
+ metadata.set(PSM0_PAGE_NUMBER, Integer.parseInt(v));
+ break;
+ case "Orientation in degrees":
+ metadata.set(PSM0_ORIENTATION, Integer.parseInt(v));
+ break;
+ case "Rotate":
+ metadata.set(PSM0_ROTATE, Integer.parseInt(v));
+ break;
+ case "Orientation confidence":
+ metadata.set(PSM0_ORIENTATION_CONFIDENCE, Double.parseDouble(v));
+ break;
+ case "Script":
+ metadata.set(PSM0_SCRIPT, v);
+ break;
+ case "Script confidence":
+ metadata.set(PSM0_SCRIPT_CONFIDENCE, Double.parseDouble(v));
+ break;
+ default:
+ LOG.warn("I regret I don't know how to parse {} with value {}", k, v);
+ }
+ }
+ line = reader.readLine();
+ }
+ }
+ }
+
private void warnOnFirstParse() {
if (!hasWarned()) {
warn();
@@ -326,18 +381,21 @@ public class TesseractOCRParser extends AbstractExternalProcessParser implements
ArrayList<String> cmd = new ArrayList<>(
Arrays.asList(getTesseractPath() + getTesseractProg(), input.getPath(),
output.getPath(), "--psm", config.getPageSegMode()));
- if (!StringUtils.isBlank(config.getLanguage())) {
- cmd.add("-l");
- cmd.add(config.getLanguage());
- }
- for (Map.Entry<String, String> entry : config.getOtherTesseractConfig().entrySet()) {
- cmd.add("-c");
- cmd.add(entry.getKey() + "=" + entry.getValue());
- }
- cmd.addAll(Arrays.asList("-c", "page_separator=" + config.getPageSeparator(), "-c",
- (config.isPreserveInterwordSpacing()) ? "preserve_interword_spaces=1" :
- "preserve_interword_spaces=0",
- config.getOutputType().name().toLowerCase(Locale.US)));
+ //if --psm == 0, don't add anything else to the command line
+ if (! "0".equals(config.getPageSegMode())) {
+ if (!StringUtils.isBlank(config.getLanguage())) {
+ cmd.add("-l");
+ cmd.add(config.getLanguage());
+ }
+ for (Map.Entry<String, String> entry : config.getOtherTesseractConfig().entrySet()) {
+ cmd.add("-c");
+ cmd.add(entry.getKey() + "=" + entry.getValue());
+ }
+ cmd.addAll(Arrays.asList("-c", "page_separator=" + config.getPageSeparator(), "-c",
+ (config.isPreserveInterwordSpacing()) ? "preserve_interword_spaces=1" :
+ "preserve_interword_spaces=0",
+ config.getOutputType().name().toLowerCase(Locale.US)));
+ }
LOG.debug("Tesseract command: " + String.join(" ", cmd));
ProcessBuilder pb = new ProcessBuilder(cmd);
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
index cc03dab..545f132 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
@@ -176,6 +176,27 @@ public class TesseractOCRParserTest extends TikaTest {
}
@Test
+ public void testPSM0() throws Exception {
+ assumeTrue(canRun(), "can run OCR");
+ //this test may be too brittle...e.g. with different versions of tesseract installed
+ try (InputStream is = getResourceAsStream("/test-configs/tika-config-psm0.xml")) {
+ TikaConfig config = new TikaConfig(is);
+ Parser p = new AutoDetectParser(config);
+ Metadata m = new Metadata();
+ getXML("testRotated+10.png", p, m);
+ assertEquals(0, m.getInt(TesseractOCRParser.PSM0_PAGE_NUMBER));
+ assertEquals(180, m.getInt(TesseractOCRParser.PSM0_ORIENTATION));
+ assertEquals(180, m.getInt(TesseractOCRParser.PSM0_ROTATE));
+ assertEquals(5.71,
+ Double.parseDouble(m.get(TesseractOCRParser.PSM0_ORIENTATION_CONFIDENCE)), 0.1);
+ assertEquals(0.83,
+ Double.parseDouble(m.get(TesseractOCRParser.PSM0_SCRIPT_CONFIDENCE)),
+ 0.1);
+ assertEquals("Latin", m.get(TesseractOCRParser.PSM0_SCRIPT));
+ }
+ }
+
+ @Test
public void testPreloadLangs() throws Exception {
assumeTrue(canRun());
TikaConfig config;
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/resources/test-configs/tika-config-psm0.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/resources/test-configs/tika-config-psm0.xml
new file mode 100644
index 0000000..c3da2b3
--- /dev/null
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/resources/test-configs/tika-config-psm0.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<properties>
+ <parsers>
+ <parser class="org.apache.tika.parser.DefaultParser">
+ <parser-exclude class="org.apache.tika.parser.ocr.TesseractOCRParser"/>
+ </parser>
+ <parser class="org.apache.tika.parser.ocr.TesseractOCRParser">
+ <params>
+ <param name="pageSegMode" type="string">0</param>
+ </params>
+ </parser>
+ </parsers>
+</properties>