You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2018/12/03 15:10:47 UTC
[tika] 02/04: TIKA-2779: Integrate/parameterize new rotated text
handling in PDFBox
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit d0567c129d7d306695cb9306da7dff552c96031d
Author: TALLISON <ta...@apache.org>
AuthorDate: Mon Dec 3 08:55:36 2018 -0500
TIKA-2779: Integrate/parameterize new rotated text handling in PDFBox
---
CHANGES.txt | 3 +
.../java/org/apache/tika/parser/pdf/PDF2XHTML.java | 136 ++++++++++++++++++---
.../java/org/apache/tika/parser/pdf/PDFParser.java | 5 +
.../apache/tika/parser/pdf/PDFParserConfig.java | 11 ++
.../apache/tika/parser/pdf/PDFParser.properties | 3 +
.../org/apache/tika/parser/pdf/PDFParserTest.java | 14 +++
.../resources/test-documents/testPDF_angles.pdf | Bin 0 -> 797493 bytes
7 files changed, 158 insertions(+), 14 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index c15d04b..92e2956 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,9 @@ Release 2.0.0 - ???
Release 1.20 - ???
+ * Integrate/parameterize new angles handling in
+ PDFBox (TIKA-2779).
+
* Upgrade to PDFBox 2.0.13 (TIKA-2788).
* Prevent content within <style/> and <script/> elements
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java b/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
index 1602a08..3893378 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
@@ -16,25 +16,14 @@
*/
package org.apache.tika.parser.pdf;
-import java.awt.image.BufferedImage;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.Writer;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
+import org.apache.pdfbox.cos.COSArray;
import org.apache.pdfbox.cos.COSBase;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.cos.COSStream;
import org.apache.pdfbox.filter.MissingImageReaderException;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.PDResources;
import org.apache.pdfbox.pdmodel.graphics.PDXObject;
import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceGray;
@@ -44,6 +33,7 @@ import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
import org.apache.pdfbox.text.PDFTextStripper;
import org.apache.pdfbox.text.TextPosition;
import org.apache.pdfbox.tools.imageio.ImageIOUtil;
+import org.apache.pdfbox.util.Matrix;
import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
import org.apache.tika.io.TikaInputStream;
@@ -55,6 +45,19 @@ import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Writer;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
/**
* Utility class that overrides the {@link PDFTextStripper} functionality
* to produce a semi-structured XHTML SAX events instead of a plain text
@@ -111,7 +114,11 @@ class PDF2XHTML extends AbstractPDF2XHTML {
// Extract text using a dummy Writer as we override the
// key methods to output to the given content
// handler.
- pdf2XHTML = new PDF2XHTML(document, handler, context, metadata, config);
+ if (config.getDetectAngles()) {
+ pdf2XHTML = new AngleDetectingPDF2XHTML(document, handler, context, metadata, config);
+ } else {
+ pdf2XHTML = new PDF2XHTML(document, handler, context, metadata, config);
+ }
config.configure(pdf2XHTML);
pdf2XHTML.writeText(document, new Writer() {
@@ -140,6 +147,82 @@ class PDF2XHTML extends AbstractPDF2XHTML {
}
}
+ private static class AngleDetectingPDF2XHTML extends PDF2XHTML {
+
+ private AngleDetectingPDF2XHTML(PDDocument document, ContentHandler handler, ParseContext context, Metadata metadata, PDFParserConfig config) throws IOException {
+ super(document, handler, context, metadata, config);
+ }
+
+ @Override
+ protected void startPage(PDPage page) throws IOException {
+ //no-op
+ }
+
+ @Override
+ protected void endPage(PDPage page) throws IOException {
+ //no-op
+ }
+
+ @Override
+ public void processPage(PDPage page) throws IOException {
+ try {
+ super.startPage(page);
+ detectAnglesAndProcessPage(page);
+ } catch (IOException e) {
+ handleCatchableIOE(e);
+ } finally {
+ super.endPage(page);
+ }
+ }
+
+ private void detectAnglesAndProcessPage(PDPage page) throws IOException {
+ //copied and pasted from https://issues.apache.org/jira/secure/attachment/12947452/ExtractAngledText.java
+ //PDFBOX-4371
+ AngleCollector angleCollector = new AngleCollector(); // alternatively, reset angles
+ angleCollector.setStartPage(getCurrentPageNo());
+ angleCollector.setEndPage(getCurrentPageNo());
+ angleCollector.getText(document);
+
+ int rotation = page.getRotation();
+ page.setRotation(0);
+
+ for (Integer angle : angleCollector.getAngles()) {
+ if (angle == 0) {
+ try {
+ super.processPage(page);
+ } catch (IOException e) {
+ handleCatchableIOE(e);
+ }
+ } else {
+ // prepend a transformation
+ try (PDPageContentStream cs = new PDPageContentStream(document, page, PDPageContentStream.AppendMode.PREPEND, false)) {
+ cs.transform(Matrix.getRotateInstance(-Math.toRadians(angle), 0, 0));
+ }
+
+ try {
+ super.processPage(page);
+ } catch (IOException e) {
+ handleCatchableIOE(e);
+ }
+
+ // remove transformation
+ COSArray contents = (COSArray) page.getCOSObject().getItem(COSName.CONTENTS);
+ contents.remove(0);
+ }
+ }
+ page.setRotation(rotation);
+ }
+ }
+
+ @Override
+ protected void processTextPosition(TextPosition text) {
+ Matrix m = text.getTextMatrix();
+ m.concatenate(text.getFont().getFontMatrix());
+ int angle = (int) Math.round(Math.toDegrees(Math.atan2(m.getShearY(), m.getScaleY())));
+ if (angle == 0) {
+ super.processTextPosition(text);
+ }
+ }
@Override
public void processPage(PDPage page) throws IOException {
@@ -366,5 +449,30 @@ class PDF2XHTML extends AbstractPDF2XHTML {
}
}
+ class AngleCollector extends PDFTextStripper {
+ Set<Integer> angles = new HashSet<>();
+
+ public Set<Integer> getAngles() {
+ return angles;
+ }
+
+ /**
+ * Instantiate a new PDFTextStripper object.
+ *
+ * @throws IOException If there is an error loading the properties.
+ */
+ AngleCollector() throws IOException {
+ }
+
+ @Override
+ protected void processTextPosition(TextPosition text) {
+ Matrix m = text.getTextMatrix();
+ m.concatenate(text.getFont().getFontMatrix());
+ int angle = (int) Math.round(Math.toDegrees(Math.atan2(m.getShearY(), m.getScaleY())));
+ angle = (angle + 360) % 360;
+ angles.add(angle);
+ }
+ }
+
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
index 43a216a..2c586fa 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
@@ -707,6 +707,11 @@ public class PDFParser extends AbstractParser implements Initializable {
}
@Field
+ void setDetectAngles(boolean detectAngles) {
+ defaultConfig.setDetectAngles(detectAngles);
+ }
+
+ @Field
void setInitializableProblemHander(String name) {
if ("ignore".equals(name)) {
setInitializableProblemHandler(InitializableProblemHandler.IGNORE);
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java b/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
index b0327af..4cdf51f 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
@@ -139,6 +139,8 @@ public class PDFParserConfig implements Serializable {
private boolean setKCMS = false;
+ private boolean detectAngles = false;
+
public PDFParserConfig() {
init(this.getClass().getResourceAsStream("PDFParser.properties"));
}
@@ -231,6 +233,7 @@ public class PDFParserConfig implements Serializable {
}
maxMainMemoryBytes = getIntProp(props.getProperty("maxMainMemoryBytes"), -1);
+ detectAngles = getBooleanProp(props.getProperty("detectAngles"), false);
}
/**
@@ -735,6 +738,14 @@ public class PDFParserConfig implements Serializable {
throw new IllegalArgumentException(sb.toString());
}
+ public void setDetectAngles(boolean detectAngles) {
+ this.detectAngles = detectAngles;
+ }
+
+ public boolean getDetectAngles() {
+ return detectAngles;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
diff --git a/tika-parsers/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties b/tika-parsers/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties
index f2d3db6..1a12ad9 100644
--- a/tika-parsers/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties
+++ b/tika-parsers/src/main/resources/org/apache/tika/parser/pdf/PDFParser.properties
@@ -39,3 +39,6 @@ ocrImageScale 2.0
maxMainMemoryBytes 524288000
#whether or not to set KCMS for faster (but legacy/unsupported) image rendering
setKCMS false
+#whether or not to add processing to detect angles and extract
+#text accordingly PDFBOX-4371
+detectAngles false
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
index 1c82ad2..d27d8bd 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
@@ -1372,6 +1372,20 @@ public class PDFParserTest extends TikaTest {
.metadata.get(TikaCoreProperties.LANGUAGE));
}
+ @Test
+ public void testAngles() throws Exception {
+ PDFParserConfig pdfParserConfig = new PDFParserConfig();
+ pdfParserConfig.setDetectAngles(true);
+ ParseContext parseContext = new ParseContext();
+ parseContext.set(PDFParserConfig.class, pdfParserConfig);
+ String xml = getXML("testPDF_angles.pdf", parseContext).xml;
+ //make sure there is only one page!
+ assertContainsCount("<div class=\"page\">", xml, 1);
+ assertContains("IN-DEMAND", xml);
+ assertContains("natural underground", xml);
+ assertContains("transport mined materials", xml);
+ }
+
/**
* Simple class to count end of document events. If functionality is useful,
* move to org.apache.tika in src/test
diff --git a/tika-parsers/src/test/resources/test-documents/testPDF_angles.pdf b/tika-parsers/src/test/resources/test-documents/testPDF_angles.pdf
new file mode 100644
index 0000000..7d29dae
Binary files /dev/null and b/tika-parsers/src/test/resources/test-documents/testPDF_angles.pdf differ