You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ti...@apache.org on 2024/12/17 18:11:02 UTC
(tika) branch branch_3x updated: TIKA-2342: suppport PDFBox IgnoreContentStreamSpaceGlyphs; add test; remove dead code line
This is an automated email from the ASF dual-hosted git repository.
tilman pushed a commit to branch branch_3x
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/branch_3x by this push:
new fb1f238a2 TIKA-2342: suppport PDFBox IgnoreContentStreamSpaceGlyphs; add test; remove dead code line
fb1f238a2 is described below
commit fb1f238a25e4be680ab92ae5684583857c33ddbe
Author: Tilman Hausherr <ti...@apache.org>
AuthorDate: Tue Dec 17 16:32:00 2024 +0100
TIKA-2342: suppport PDFBox IgnoreContentStreamSpaceGlyphs; add test; remove dead code line
---
.../java/org/apache/tika/parser/pdf/PDFParser.java | 18 +++++++++++
.../apache/tika/parser/pdf/PDFParserConfig.java | 23 ++++++++++++++
.../org/apache/tika/parser/pdf/PDFParserTest.java | 33 ++++++++++++++++++++-
.../testContentStreamSpaceGlyphs.pdf | Bin 0 -> 805 bytes
4 files changed, 73 insertions(+), 1 deletion(-)
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
index 77ccb9231..13d09e604 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
@@ -759,6 +759,24 @@ public class PDFParser implements Parser, RenderingParser, Initializable {
return defaultConfig.isSuppressDuplicateOverlappingText();
}
+ /**
+ * If true, the parser should ignore spaces in the content stream and rely purely on the
+ * algorithm to determine where word breaks are (PDFBOX-3774). This can improve text extraction
+ * results where the content stream is sorted by position and has text overlapping spaces, but
+ * could cause some word breaks to not be added to the output. By default this is disabled.
+ */
+ @Field
+ public void setIgnoreContentStreamSpaceGlyphs(boolean v) {
+ defaultConfig.setIgnoreContentStreamSpaceGlyphs(v);
+ }
+
+ /**
+ * @see #setIgnoreContentStreamSpaceGlyphs(boolean)
+ */
+ public boolean isIgnoreContentStreamSpaceGlyphs() {
+ return defaultConfig.isIgnoreContentStreamSpaceGlyphs();
+ }
+
/**
* If true, the parser should try to remove duplicated
* text over the same region. This is needed for some
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
index d1f93fc41..af6213ba9 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
@@ -65,6 +65,9 @@ public class PDFParserConfig implements Serializable {
// True if we let PDFBox remove duplicate overlapping text:
private boolean suppressDuplicateOverlappingText = false;
+ // True if we let PDFBox ignore spaces in the content stream and rely purely on the algorithm:
+ private boolean ignoreContentStreamSpaceGlyphs = false;
+
// True if we extract annotation text ourselves
// (workaround for PDFBOX-1143):
private boolean extractAnnotationText = true;
@@ -223,6 +226,8 @@ public class PDFParserConfig implements Serializable {
pdf2XHTML.setDropThreshold(dropThreshold);
}
pdf2XHTML.setSuppressDuplicateOverlappingText(isSuppressDuplicateOverlappingText());
+ // TODO TIKA-2342 activate after PDFBox release
+ //pdf2XHTML.setIgnoreContentStreamSpaceGlyphs(isIgnoreContentStreamSpaceGlyphs());
}
/**
@@ -404,6 +409,24 @@ public class PDFParserConfig implements Serializable {
userConfigured.add("suppressDuplicateOverlappingText");
}
+ /**
+ * @see #setIgnoreContentStreamSpaceGlyphs(boolean)
+ */
+ public boolean isIgnoreContentStreamSpaceGlyphs() {
+ return ignoreContentStreamSpaceGlyphs;
+ }
+
+ /**
+ * If true, the parser should ignore spaces in the content stream and rely purely on the
+ * algorithm to determine where word breaks are (PDFBOX-3774). This can improve text extraction
+ * results where the content stream is sorted by position and has text overlapping spaces, but
+ * could cause some word breaks to not be added to the output. By default this is disabled.
+ */
+ public void setIgnoreContentStreamSpaceGlyphs(boolean ignoreContentStreamSpaceGlyphs) {
+ this.ignoreContentStreamSpaceGlyphs = ignoreContentStreamSpaceGlyphs;
+ userConfigured.add("ignoreContentStreamSpaceGlyphs");
+ }
+
/**
* @see #setExtractAnnotationText(boolean)
*/
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
index e3cfe5415..b4c22d061 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
@@ -465,6 +465,37 @@ public class PDFParserTest extends TikaTest {
}
+ // TODO TIKA-2342 activate after PDFBox release
+ // @Test
+ public void testIgnoreContentStreamSpaceGlyphs() throws Exception {
+ PDFParser parser = new PDFParser();
+ // Default is false (keep spaces, don't sort):
+ XMLResult r = getXML("testContentStreamSpaceGlyphs.pdf", parser);
+ assertContains("( )overlap", r.xml);
+
+ parser.getPDFParserConfig().setIgnoreContentStreamSpaceGlyphs(true);
+ r = getXML("testContentStreamSpaceGlyphs.pdf", parser);
+ assertContains("( )overlap", r.xml);
+ parser.getPDFParserConfig().setSortByPosition(true);
+ r = getXML("testContentStreamSpaceGlyphs.pdf", parser);
+ assertContains("( overlap )", r.xml);
+
+ //now try with autodetect
+ ParseContext context = new ParseContext();
+ PDFParserConfig config = new PDFParserConfig();
+ context.set(PDFParserConfig.class, config);
+ r = getXML("testContentStreamSpaceGlyphs.pdf", context);
+ // Default is false (keep spaces, don't sort):
+ assertContains("( )overlap", r.xml);
+
+ config.setIgnoreContentStreamSpaceGlyphs(true);
+ r = getXML("testContentStreamSpaceGlyphs.pdf", context);
+ assertContains("( )overlap", r.xml);
+ config.setSortByPosition(true);
+ r = getXML("testContentStreamSpaceGlyphs.pdf", context);
+ assertContains("( overlap )", r.xml);
+ }
+
@Test
public void testSortByPosition() throws Exception {
PDFParser parser = new PDFParser();
@@ -499,7 +530,7 @@ public class PDFParserTest extends TikaTest {
config.setSortByPosition(true);
context.set(PDFParserConfig.class, config);
- stream = getResourceAsStream("/test-documents/testPDFTwoTextBoxes.pdf");
+ //stream = getResourceAsStream("/test-documents/testPDFTwoTextBoxes.pdf");
content = getText("testPDFTwoTextBoxes.pdf", new Metadata(), context);
content = content.replaceAll("\\s+", " ");
// Column text is now interleaved:
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/resources/test-documents/testContentStreamSpaceGlyphs.pdf b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/resources/test-documents/testContentStreamSpaceGlyphs.pdf
new file mode 100644
index 000000000..1a73dc4e2
Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/resources/test-documents/testContentStreamSpaceGlyphs.pdf differ