You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2021/01/05 18:55:53 UTC
[tika] branch main updated (476ed94 -> 69142fe)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git.
from 476ed94 Simplify method CompressorParser#parse
new 1798e80 TIKA-3262
new 69142fe TIKA-3260 -- update rotation.py to work with python3 and modern version of dependencies. Fix cross-platform problems with Windows.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/tika/utils/CompareUtils.java | 2 +-
.../apache/tika/utils/ServiceLoaderUtilsTest.java | 6 +-
.../apache/tika/parser/ocr/TesseractOCRConfig.java | 26 +++++-
.../apache/tika/parser/ocr/TesseractOCRParser.java | 94 +++++++++++++++-------
.../tika/parser/ocr/TesseractOCRConfig.properties | 1 +
.../org/apache/tika/parser/ocr/rotation.py | 13 +--
.../apache/tika/detect/TestDetectorLoading.java | 2 +-
.../tika/parser/ocr/TesseractOCRParserTest.java | 5 +-
.../apache/tika/utils/ServiceLoaderUtilsTest.java | 2 +-
9 files changed, 104 insertions(+), 47 deletions(-)
[tika] 02/02: TIKA-3260 -- update rotation.py to work with python3
and modern version of dependencies. Fix cross-platform problems with
Windows.
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 69142fe18ebadfc3110476a190b85d796a5c6a5e
Author: tallison <ta...@apache.org>
AuthorDate: Tue Jan 5 13:55:31 2021 -0500
TIKA-3260 -- update rotation.py to work with python3 and modern version of dependencies. Fix cross-platform problems with Windows.
---
.../apache/tika/parser/ocr/TesseractOCRConfig.java | 26 +++++-
.../apache/tika/parser/ocr/TesseractOCRParser.java | 94 +++++++++++++++-------
.../tika/parser/ocr/TesseractOCRConfig.properties | 1 +
.../org/apache/tika/parser/ocr/rotation.py | 13 +--
.../tika/parser/ocr/TesseractOCRParserTest.java | 5 +-
5 files changed, 98 insertions(+), 41 deletions(-)
diff --git a/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRConfig.java b/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRConfig.java
index 7b11a7d..de7418b 100644
--- a/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRConfig.java
+++ b/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRConfig.java
@@ -88,6 +88,9 @@ public class TesseractOCRConfig implements Serializable {
// Path to ImageMagick program, if not on system path.
private String imageMagickPath = "";
+ // Path to the python3 executable, if not on system path
+ private String pythonPath = "";
+
// resolution of processed image (in dpi).
private int density = 300;
@@ -101,6 +104,8 @@ public class TesseractOCRConfig implements Serializable {
private String filter = "triangle";
// factor by which image is to be scaled.
+ // TODO: we should make this dynamic depending on the size of the image
+ // The current testRotation.png takes minutes to expand 900%
private int resize = 900;
// See setPageSeparator.
@@ -188,6 +193,9 @@ public class TesseractOCRConfig implements Serializable {
setApplyRotation(
getProp(props, "applyRotation", getApplyRotation()));
+ setPythonPath(
+ getProp(props, "pythonPath", getPythonPath()));
+
loadOtherTesseractConfig(props);
}
@@ -540,13 +548,27 @@ public class TesseractOCRConfig implements Serializable {
*/
public void setImageMagickPath(String imageMagickPath) {
imageMagickPath = FilenameUtils.normalize(imageMagickPath);
- if (!imageMagickPath.isEmpty() && !imageMagickPath.endsWith(File.separator))
+ if (!imageMagickPath.isEmpty() && !imageMagickPath.endsWith(File.separator)) {
imageMagickPath += File.separator;
-
+ }
this.imageMagickPath = imageMagickPath;
}
/**
+ * Path to the directory that contains the Python executable.
+ * As of 2.0.0, Tika expects the executable python3 or python3.exe
+ * to be in the directory specified by the pythonPath
+ *
+ * @param pythonPath
+ */
+ public void setPythonPath(String pythonPath) {
+ this.pythonPath = FilenameUtils.normalize(pythonPath);
+ }
+
+ public String getPythonPath() {
+ return pythonPath;
+ }
+ /**
* @return Whether or not a rotation value should be calculated and passed to ImageMagick before performing OCR.
* (Requires that Python is installed).
*/
diff --git a/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java b/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
index ee4b81a..ff2ea2f 100644
--- a/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
+++ b/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
@@ -43,6 +43,7 @@ import org.apache.tika.parser.image.TiffParser;
import org.apache.tika.parser.image.JpegParser;
import org.apache.tika.sax.OfflineContentHandler;
import org.apache.tika.sax.XHTMLContentHandler;
+import org.apache.tika.utils.ProcessUtils;
import org.apache.tika.utils.XMLReaderUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -117,6 +118,7 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
private static Map<String,Boolean> TESSERACT_PRESENT = new HashMap<>();
private static Map<String,Boolean> IMAGE_MAGICK_PRESENT = new HashMap<>();
+ private static Map<String, Boolean> PYTHON_PRESENT = new HashMap<>();
@Override
@@ -188,11 +190,6 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
IMAGE_MAGICK_PRESENT.put(ImageMagick, false);
return false;
}
- if (SystemUtils.IS_OS_WINDOWS && config.getImageMagickPath().isEmpty()) {
- LOG.warn("Must specify path for imagemagick on Windows OS to avoid accidental confusion with convert.exe");
- IMAGE_MAGICK_PRESENT.put(ImageMagick, false);
- return false;
- }
// Try running ImageMagick program from there, and see if it exists + works
String[] checkCmd = { ImageMagick };
@@ -203,28 +200,50 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
}
+
private String getImageMagickPath(TesseractOCRConfig config) {
return config.getImageMagickPath() + getImageMagickProg();
}
- public static boolean hasPython() {
+ private static String getPythonPath(TesseractOCRConfig config) {
+ return config.getPythonPath()+getPythonProg();
+ }
+
+ public static boolean hasPython(TesseractOCRConfig config) {
+ String pythonPath = getPythonPath(config);
+ if (PYTHON_PRESENT.containsKey(pythonPath)) {
+ return PYTHON_PRESENT.get(pythonPath);
+ }
+ //prevent memory bloat
+ if (PYTHON_PRESENT.size() > 100) {
+ PYTHON_PRESENT.clear();
+ }
+ //check that directory exists
+ if (!config.getPythonPath().isEmpty() &&
+ ! Files.isDirectory(Paths.get(config.getPythonPath()))) {
+ PYTHON_PRESENT.put(pythonPath, false);
+ return false;
+ }
+
// check if python is installed and it has the required dependencies for the rotation program to run
boolean hasPython = false;
TemporaryResources tmp = null;
try {
tmp = new TemporaryResources();
File importCheck = tmp.createTemporaryFile();
- String prg = "import numpy, matplotlib, skimage, _tkinter";
+ String prg = "from skimage.transform import radon\n" +
+ "from PIL import Image\n"+"" +
+ "import numpy\n";
OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(importCheck), Charset.forName("UTF-8"));
out.write(prg);
out.close();
- Process p = Runtime.getRuntime().exec("python " + importCheck.getAbsolutePath());
+ Process p = Runtime.getRuntime().exec(new String[]{
+ pythonPath,
+ ProcessUtils.escapeCommandLine(importCheck.getAbsolutePath())});
if (p.waitFor() == 0) {
hasPython = true;
}
-
-
} catch (SecurityException e) {
throw e;
} catch (Exception e) {
@@ -232,7 +251,7 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
} finally {
IOUtils.closeQuietly(tmp);
}
-
+ PYTHON_PRESENT.put(pythonPath, hasPython);
return hasPython;
}
@@ -356,22 +375,21 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
Files.copy(in, rotationScript.toPath(), StandardCopyOption.REPLACE_EXISTING);
}
- CommandLine commandLine = new CommandLine("python");
- String[] args = {"-W",
- "ignore",
- rotationScript.getAbsolutePath(),
- "-f",
- scratchFile.getAbsolutePath()};
- commandLine.addArguments(args, true);
- String angle = "0";
-
- DefaultExecutor executor = new DefaultExecutor();
- ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
- PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream);
- executor.setStreamHandler(streamHandler);
-
+ String angle = "0";
+ DefaultExecutor executor = new DefaultExecutor();
// determine the angle of rotation required to make the text horizontal
- if(config.getApplyRotation() && hasPython()) {
+ if(config.getApplyRotation() && hasPython(config)) {
+ CommandLine commandLine = new CommandLine(getPythonPath(config));
+ String[] args = {"-W",
+ "ignore",
+ rotationScript.getAbsolutePath(),
+ "-f",
+ scratchFile.getAbsolutePath()};
+ commandLine.addArguments(args, true);
+
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream);
+ executor.setStreamHandler(streamHandler);
try {
executor.execute(commandLine);
String tmpAngle = outputStream.toString("UTF-8").trim();
@@ -379,13 +397,16 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
Double.parseDouble(tmpAngle);
angle = tmpAngle;
} catch(Exception e) {
-
+ //TODO: log
}
}
// process the image - parameter values can be set in TesseractOCRConfig.properties
- commandLine = new CommandLine(getImageMagickPath(config));
- args = new String[]{
+ CommandLine commandLine = new CommandLine(getImageMagickPath(config));
+ if (System.getProperty("os.name").startsWith("Windows")) {
+ commandLine.addArgument("convert");
+ }
+ String[] args = new String[]{
"-density", Integer.toString(config.getDensity()),
"-depth ", Integer.toString(config.getDepth()),
"-colorspace", config.getColorspace(),
@@ -396,10 +417,11 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
scratchFile.getAbsolutePath()
};
commandLine.addArguments(args, true);
+ LOG.debug("ImageMagick commandline: "+commandLine);
try {
executor.execute(commandLine);
} catch(Exception e) {
-
+ //TODO: log
}
tmp.close();
@@ -633,7 +655,12 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
}
public static String getImageMagickProg() {
- return System.getProperty("os.name").startsWith("Windows") ? "convert.exe" : "convert";
+ return System.getProperty("os.name").startsWith("Windows") ?
+ "magick" : "convert";
+ }
+
+ public static String getPythonProg() {
+ return "python3";
}
@@ -785,6 +812,11 @@ public class TesseractOCRParser extends AbstractParser implements Initializable
defaultConfig.setApplyRotation(applyRotation);
}
+ @Field
+ public void setPythonPath(String pythonPath) {
+ defaultConfig.setPythonPath(pythonPath);
+ }
+
public TesseractOCRConfig getDefaultConfig() {
return defaultConfig;
}
diff --git a/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/resources/org/apache/tika/parser/ocr/TesseractOCRConfig.properties b/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/resources/org/apache/tika/parser/ocr/TesseractOCRConfig.properties
index 73c9083..ea17edc 100644
--- a/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/resources/org/apache/tika/parser/ocr/TesseractOCRConfig.properties
+++ b/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/resources/org/apache/tika/parser/ocr/TesseractOCRConfig.properties
@@ -28,6 +28,7 @@ preserveInterwordSpacing=false
# to enable processing, set enableImageProcessing to 1
enableImageProcessing=0
ImageMagickPath=
+pythonPath=
density=300
depth=4
colorspace=gray
diff --git a/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/resources/org/apache/tika/parser/ocr/rotation.py b/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/resources/org/apache/tika/parser/ocr/rotation.py
index c619325..c04f699 100644
--- a/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/resources/org/apache/tika/parser/ocr/rotation.py
+++ b/tika-parsers/tika-parsers-classic/tika-parsers-classic-modules/tika-parser-ocr-module/src/main/resources/org/apache/tika/parser/ocr/rotation.py
@@ -15,14 +15,9 @@
limitations under the License.
"""
-from __future__ import division, print_function
-import numpy
from skimage.transform import radon
from PIL import Image
-from numpy import asarray, mean, array, blackman
-from numpy.fft import rfft
-import matplotlib.pyplot as plt
-from matplotlib.mlab import rms_flat
+from numpy import asarray, mean, array, blackman, sqrt, absolute
import sys
import getopt
@@ -68,5 +63,11 @@ def main(argv):
print('{:.2f}'.format(-(90-rotation)))
+def rms_flat(a):
+ """
+ Return the root mean square of all the elements of *a*, flattened out.
+ """
+ return sqrt(mean(absolute(a)**2))
+
if __name__ == "__main__":
main(sys.argv[1:])
diff --git a/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java b/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
index f9bceff..3f1feb9 100644
--- a/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
+++ b/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/ocr/TesseractOCRParserTest.java
@@ -289,10 +289,11 @@ public class TesseractOCRParserTest extends TikaTest {
@Test
public void testRotatedOCR() throws Exception {
- if (TesseractOCRParser.hasPython()) {
- TesseractOCRConfig config = new TesseractOCRConfig();
+ TesseractOCRConfig config = new TesseractOCRConfig();
+ if (TesseractOCRParser.hasPython(config)) {
config.setApplyRotation(true);
config.setEnableImageProcessing(1);
+ config.setResize(100);
ParseContext parseContext = new ParseContext();
parseContext.set(TesseractOCRConfig.class, config);
assumeTrue(canRun(config));