You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by mi...@apache.org on 2011/10/20 14:49:51 UTC
svn commit: r1186771 - in /tika/trunk: ./
tika-parsers/src/main/java/org/apache/tika/parser/pdf/
tika-parsers/src/test/java/org/apache/tika/parser/pdf/
Author: mikemccand
Date: Thu Oct 20 12:49:50 2011
New Revision: 1186771
URL: http://svn.apache.org/viewvc?rev=1186771&view=rev
Log:
TIKA-738: optionally extract PDF annotations
Modified:
tika/trunk/CHANGES.txt
tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
Modified: tika/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/tika/trunk/CHANGES.txt?rev=1186771&r1=1186770&r2=1186771&view=diff
==============================================================================
--- tika/trunk/CHANGES.txt (original)
+++ tika/trunk/CHANGES.txt Thu Oct 20 12:49:50 2011
@@ -17,6 +17,8 @@ Release 0.11 - Current Development
* TIKA-753: Improve performance when extracting embedded office docs.
+ * TIKA-738: Optionally extract text from PDF annotations.
+
Release 0.10 - 09/25/2011
The most notable changes in Tika 0.10 over previous releases are:
Modified: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java?rev=1186771&r1=1186770&r2=1186771&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java (original)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java Thu Oct 20 12:49:50 2011
@@ -22,6 +22,8 @@ import java.io.Writer;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.util.PDFTextStripper;
+import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
+import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationMarkup;
import org.apache.pdfbox.util.TextPosition;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.IOExceptionWithCause;
@@ -51,12 +53,12 @@ class PDF2XHTML extends PDFTextStripper
* @throws TikaException if the PDF document can not be processed
*/
public static void process(
- PDDocument document, ContentHandler handler, Metadata metadata)
+ PDDocument document, ContentHandler handler, Metadata metadata, boolean extractAnnotationText)
throws SAXException, TikaException {
try {
// Extract text using a dummy Writer as we override the
// key methods to output to the given content handler.
- new PDF2XHTML(handler, metadata).writeText(document, new Writer() {
+ new PDF2XHTML(handler, metadata, extractAnnotationText).writeText(document, new Writer() {
@Override
public void write(char[] cbuf, int off, int len) {
}
@@ -77,10 +79,12 @@ class PDF2XHTML extends PDFTextStripper
}
private final XHTMLContentHandler handler;
+ private final boolean extractAnnotationText;
- private PDF2XHTML(ContentHandler handler, Metadata metadata)
+ private PDF2XHTML(ContentHandler handler, Metadata metadata, boolean extractAnnotationText)
throws IOException {
this.handler = new XHTMLContentHandler(handler, metadata);
+ this.extractAnnotationText = extractAnnotationText;
setForceParsing(true);
setSortByPosition(false);
}
@@ -115,8 +119,51 @@ class PDF2XHTML extends PDFTextStripper
@Override
protected void endPage(PDPage page) throws IOException {
+
try {
+ // TODO: remove once PDFBOX-1143 is fixed:
handler.endElement("p");
+ if (extractAnnotationText) {
+ boolean foundTextAnnots = false;
+ for(Object o : page.getAnnotations()) {
+ if ((o instanceof PDAnnotation) && PDAnnotationMarkup.SUB_TYPE_FREETEXT.equals(((PDAnnotation) o).getSubtype())) {
+ // It's a text annotation:
+ PDAnnotationMarkup annot = (PDAnnotationMarkup) o;
+ String title = annot.getTitlePopup();
+ String subject = annot.getTitlePopup();
+ String contents = annot.getContents();
+ // TODO: maybe also annot.getRichContents()?
+ if (title != null || subject != null || contents != null) {
+ if (!foundTextAnnots) {
+ handler.endElement("p");
+ foundTextAnnots = true;
+ }
+
+ handler.startElement("div", "class", "annotation");
+
+ if (title != null) {
+ handler.startElement("div", "class", "annotationTitle");
+ handler.characters(title);
+ handler.endElement("div");
+ }
+
+ if (subject != null) {
+ handler.startElement("div", "class", "annotationSubject");
+ handler.characters(subject);
+ handler.endElement("div");
+ }
+
+ if (contents != null) {
+ handler.startElement("div", "class", "annotationContents");
+ handler.characters(contents);
+ handler.endElement("div");
+ }
+
+ handler.endElement("div");
+ }
+ }
+ }
+ }
handler.endElement("div");
} catch (SAXException e) {
throw new IOExceptionWithCause("Unable to end a page", e);
Modified: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java?rev=1186771&r1=1186770&r2=1186771&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java (original)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pdf/PDFParser.java Thu Oct 20 12:49:50 2011
@@ -61,6 +61,8 @@ public class PDFParser extends AbstractP
*/
public static final String PASSWORD = "org.apache.tika.parser.pdf.password";
+ private boolean extractAnnotationText = true;
+
private static final Set<MediaType> SUPPORTED_TYPES =
Collections.singleton(MediaType.application("pdf"));
@@ -88,7 +90,7 @@ public class PDFParser extends AbstractP
}
metadata.set(Metadata.CONTENT_TYPE, "application/pdf");
extractMetadata(pdfDocument, metadata);
- PDF2XHTML.process(pdfDocument, handler, metadata);
+ PDF2XHTML.process(pdfDocument, handler, metadata, extractAnnotationText);
} finally {
pdfDocument.close();
}
@@ -165,4 +167,19 @@ public class PDFParser extends AbstractP
addMetadata(metadata, name, value.toString());
}
}
+
+ /**
+ * If true (the default), text in annotations will be
+ * extracted.
+ */
+ public void setExtractAnnotationText(boolean v) {
+ extractAnnotationText = v;
+ }
+
+ /**
+ * If true, text in annotations will be extracted.
+ */
+ public boolean getExtractAnnotationText() {
+ return extractAnnotationText;
+ }
}
Modified: tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java?rev=1186771&r1=1186770&r2=1186771&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java (original)
+++ tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java Thu Oct 20 12:49:50 2011
@@ -218,8 +218,7 @@ public class PDFParserTest extends TikaT
//assertContains("\uD800\uDF32\uD800\uDF3f\uD800\uDF44\uD800\uDF39\uD800\uDF43\uD800\uDF3A", content);
}
- // TIKA-738: re-enable this
- public void IGNOREtestAnnotations() throws Exception {
+ public void testAnnotations() throws Exception {
Parser parser = new AutoDetectParser(); // Should auto-detect!
ContentHandler handler = new BodyContentHandler();
Metadata metadata = new Metadata();
@@ -234,6 +233,23 @@ public class PDFParserTest extends TikaT
content = content.replaceAll("[\\s\u00a0]+"," ");
assertContains("Here is some text", content);
assertContains("Here is a comment", content);
+
+ // Test w/ annotation text disabled:
+ PDFParser pdfParser = new PDFParser();
+ pdfParser.setExtractAnnotationText(false);
+ handler = new BodyContentHandler();
+ metadata = new Metadata();
+ context = new ParseContext();
+ stream = getResourceAsStream("/test-documents/testAnnotations.pdf");
+ try {
+ pdfParser.parse(stream, handler, metadata, context);
+ } finally {
+ stream.close();
+ }
+ content = handler.toString();
+ content = content.replaceAll("[\\s\u00a0]+"," ");
+ assertContains("Here is some text", content);
+ assertEquals(-1, content.indexOf("Here is a comment"));
}
public void testPageNumber() throws Exception {