You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2021/10/20 18:11:35 UTC
[tika] branch main updated: TIKA-3580 -- improve metadata handling
in the PDFParser, and improve documentation for table structures.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 534c269 TIKA-3580 -- improve metadata handling in the PDFParser, and improve documentation for table structures.
534c269 is described below
commit 534c269a1afc9f29daa64471a19222d818829511
Author: tallison <ta...@apache.org>
AuthorDate: Wed Oct 20 14:11:18 2021 -0400
TIKA-3580 -- improve metadata handling in the PDFParser, and improve documentation for table structures.
---
.../java/org/apache/tika/parser/pdf/PDFParser.java | 10 +++++-----
.../apache/tika/parser/pdf/PDMetadataExtractor.java | 20 +++++++++++++++++++-
.../org/apache/tika/parser/pdf/PDFParserTest.java | 11 +++++------
3 files changed, 29 insertions(+), 12 deletions(-)
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
index b4801f2..3835179 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
@@ -57,8 +57,6 @@ import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.AccessPermissions;
import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.Office;
-import org.apache.tika.metadata.OfficeOpenXMLCore;
import org.apache.tika.metadata.PDF;
import org.apache.tika.metadata.PagedText;
import org.apache.tika.metadata.TikaCoreProperties;
@@ -86,13 +84,17 @@ import org.apache.tika.sax.XHTMLContentHandler;
* turn this feature on, see
* {@link PDFParserConfig#setExtractInlineImages(boolean)}.
* <p/>
- * Please note that tables are not stored as entities within PDFs. It
+ * Please note that many pdfs do not store table structures.
+ * So you should not expect table markup for what looks like a table. It
* takes significant computation to identify and then correctly extract
* tables from PDFs. As of this writing, the {@link PDFParser} extracts
* text within tables, but it does not compute table cell boundaries or
* table row boundaries. Please see
* <a href="http://tabula.technology/">tabula</a> for one project that
* tries to maintain the structure of tables represented in PDFs.
+ *
+ * If your PDFs contain marked content or tags, consider
+ * {@link PDFParserConfig#setExtractMarkedContent(boolean)}
*/
public class PDFParser extends AbstractParser implements Initializable {
@@ -292,7 +294,6 @@ public class PDFParser extends AbstractParser implements Initializable {
PDMetadataExtractor.addMetadata(metadata, PDF.DOC_INFO_CREATOR_TOOL, info.getCreator());
PDMetadataExtractor
.addMetadata(metadata, TikaCoreProperties.CREATOR_TOOL, info.getCreator());
- PDMetadataExtractor.addMetadata(metadata, Office.KEYWORDS, info.getKeywords());
PDMetadataExtractor.addMetadata(metadata, PDF.DOC_INFO_KEY_WORDS, info.getKeywords());
PDMetadataExtractor.addMetadata(metadata, PDF.DOC_INFO_PRODUCER, info.getProducer());
PDMetadataExtractor.addMetadata(metadata, PDF.PRODUCER, info.getProducer());
@@ -301,7 +302,6 @@ public class PDFParser extends AbstractParser implements Initializable {
PDMetadataExtractor.addMetadata(metadata, TikaCoreProperties.SUBJECT, info.getKeywords());
PDMetadataExtractor.addMetadata(metadata, TikaCoreProperties.SUBJECT, info.getSubject());
- PDMetadataExtractor.addMetadata(metadata, OfficeOpenXMLCore.SUBJECT, info.getSubject());
PDMetadataExtractor.addMetadata(metadata, PDF.DOC_INFO_TRAPPED, info.getTrapped());
Calendar created = info.getCreationDate();
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDMetadataExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDMetadataExtractor.java
index 9ce2aae..5a3f85c 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDMetadataExtractor.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDMetadataExtractor.java
@@ -49,6 +49,7 @@ import org.apache.tika.metadata.XMP;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.xmp.JempboxExtractor;
+import org.apache.tika.utils.StringUtils;
import org.apache.tika.utils.XMLReaderUtils;
class PDMetadataExtractor {
@@ -301,7 +302,15 @@ class PDMetadataExtractor {
static void addMetadata(Metadata metadata, Property property, String value) {
if (value != null) {
String decoded = decode(value);
+ if (StringUtils.isBlank(decoded)) {
+ return;
+ }
if (property.isMultiValuePermitted() || metadata.get(property) == null) {
+ for (String v : metadata.getValues(property)) {
+ if (v.equals(decoded)) {
+ return;
+ }
+ }
metadata.add(property, decoded);
}
//silently skip adding property that already exists if multiple values are not permitted
@@ -310,7 +319,16 @@ class PDMetadataExtractor {
static void addMetadata(Metadata metadata, String name, String value) {
if (value != null) {
- metadata.add(name, decode(value));
+ String decoded = decode(value);
+ if (StringUtils.isBlank(decoded)) {
+ return;
+ }
+ for (String v : metadata.getValues(name)) {
+ if (v.equals(decoded)) {
+ return;
+ }
+ }
+ metadata.add(name, decoded);
}
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
index d765e1f..0749264 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
@@ -53,8 +53,6 @@ import org.apache.tika.exception.ZeroByteFileException;
import org.apache.tika.extractor.DocumentSelector;
import org.apache.tika.metadata.Font;
import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.Office;
-import org.apache.tika.metadata.OfficeOpenXMLCore;
import org.apache.tika.metadata.PDF;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.metadata.XMP;
@@ -180,7 +178,7 @@ public class PDFParserTest extends TikaTest {
assertEquals("application/pdf", metadata.get(Metadata.CONTENT_TYPE));
assertEquals("The Bank of England", metadata.get(TikaCoreProperties.CREATOR));
assertEquals("Speeches by Andrew G Haldane",
- metadata.get(OfficeOpenXMLCore.SUBJECT));
+ metadata.get(TikaCoreProperties.SUBJECT));
assertEquals(
"Rethinking the Financial Network, Speech by Andrew G Haldane, " +
"Executive Director, Financial Stability " +
@@ -206,7 +204,7 @@ public class PDFParserTest extends TikaTest {
assertEquals("application/pdf", metadata.get(Metadata.CONTENT_TYPE));
assertEquals("The Bank of England", metadata.get(TikaCoreProperties.CREATOR));
- assertEquals("Speeches by Andrew G Haldane", metadata.get(OfficeOpenXMLCore.SUBJECT));
+ assertEquals("Speeches by Andrew G Haldane", metadata.get(TikaCoreProperties.SUBJECT));
assertEquals(
"Rethinking the Financial Network, Speech by Andrew G Haldane, " +
"Executive Director, Financial Stability delivered at the " +
@@ -299,10 +297,11 @@ public class PDFParserTest extends TikaTest {
}
assertContains("Keyword1 Keyword2", content);
- assertEquals("Keyword1 Keyword2", metadata.get(Office.KEYWORDS));
+ assertContains("Keyword1 Keyword2", metadata.get(TikaCoreProperties.SUBJECT));
assertContains("Subject is here", content);
- assertEquals("Subject is here", metadata.get(OfficeOpenXMLCore.SUBJECT));
+ assertContains("Subject is here",
+ Arrays.asList(metadata.getValues(TikaCoreProperties.SUBJECT)));
assertContains("Suddenly some Japanese text:", content);
// Special version of (GHQ)